diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml deleted file mode 100644 index 176ee6e86db25..0000000000000 --- a/.github/workflows/cleanup.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: cleanup - -on: - schedule: - - cron: '*/5 * * * *' - -jobs: - cancel: - runs-on: ubuntu-latest - steps: - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: hive-tests.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: kudu.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: maven-checks.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: product-tests-basic-environment.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: product-tests-specific-environment.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: spark-integration.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: tests.yml - - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e - with: - token: ${{ github.token }} - workflow: web-ui-checks.yml diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml index 73926a0be725f..b97a1f3cf24c4 100644 --- a/.github/workflows/hive-tests.yml +++ b/.github/workflows/hive-tests.yml @@ -31,12 +31,16 @@ jobs: - '!presto-docs/**' hive-tests: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-hive-tests-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-hive-tests-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -45,7 +49,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs @@ -75,12 +79,16 @@ jobs: fi hive-dockerized-tests: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 20 concurrency: - group: ${{ github.workflow }}-hive-dockerized-tests-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-hive-dockerized-tests-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -89,7 +97,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/kudu.yml b/.github/workflows/kudu.yml index 1616df0f10d15..8d1c4151ef809 100644 --- a/.github/workflows/kudu.yml +++ b/.github/workflows/kudu.yml @@ -29,12 +29,16 @@ jobs: codechange: - '!presto-docs/**' kudu: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-kudu-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-kudu-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -43,7 +47,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index ff7403f8f39e1..629239f8f3100 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -12,10 +12,14 @@ env: jobs: maven-checks: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest timeout-minutes: 45 concurrency: - group: ${{ github.workflow }}-maven-checks-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-maven-checks-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - name: Free Disk Space @@ -29,7 +33,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/product-tests-basic-environment.yml b/.github/workflows/product-tests-basic-environment.yml index 26eb105b61794..5d44eba7bb1f2 100644 --- a/.github/workflows/product-tests-basic-environment.yml +++ b/.github/workflows/product-tests-basic-environment.yml @@ -29,12 +29,16 @@ jobs: codechange: - '!presto-docs/**' product-tests-basic-environment: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-product-tests-basic-environment-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-product-tests-basic-environment-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - name: Free Disk Space @@ -49,7 +53,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs @@ -58,4 +62,6 @@ jobs: export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing,!presto-test-coverage' - name: Run Product Tests Basic Environment + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh multinode -x quarantine,big_query,storage_formats,profile_specific_tests,tpcds,cassandra,mysql_connector,postgresql_connector,mysql,kafka,avro diff --git a/.github/workflows/product-tests-specific-environment.yml b/.github/workflows/product-tests-specific-environment.yml index fc5647ebd2b17..0a29cec164958 100644 --- a/.github/workflows/product-tests-specific-environment.yml +++ b/.github/workflows/product-tests-specific-environment.yml @@ -29,12 +29,16 @@ jobs: codechange: - '!presto-docs/**' product-tests-specific-environment1: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-product-tests-specific-environment1-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-product-tests-specific-environment1-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - name: Free Disk Space @@ -49,7 +53,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs @@ -58,26 +62,40 @@ jobs: export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing,!presto-test-coverage' - name: Product Tests Specific 1.1 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode -g hdfs_no_impersonation,avro - name: Product Tests Specific 1.2 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-no-impersonation -g hdfs_no_impersonation # temporarily disable this flaky run. see issue #20388 for details # - name: Product Tests Specific 1.3 # run: presto-product-tests/bin/run_on_docker.sh singlenode-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation - name: Product Tests Specific 1.4 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation,authorization,hive_file_header - name: Product Tests Specific 1.5 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation-cross-realm -g storage_formats,cli,hdfs_impersonation - name: Product Tests Specific 1.6 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh multinode-tls-kerberos -g cli,group-by,join,tls product-tests-specific-environment2: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-product-tests-specific-environment2-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-product-tests-specific-environment2-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - name: Free Disk Space @@ -92,7 +110,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs @@ -101,19 +119,33 @@ jobs: export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing,!presto-test-coverage' - name: Product Tests Specific 2.1 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-ldap -g ldap -x simba_jdbc - name: Product Tests Specific 2.2 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh multinode-tls -g smoke,cli,group-by,join,tls - name: Product Tests Specific 2.3 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-mysql -g mysql_connector,mysql - name: Product Tests Specific 2.4 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-postgresql -g postgresql_connector - name: Product Tests Specific 2.5 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-cassandra -g cassandra # temporarily disable this flaky run. see issue #20388 for details # - name: Product Tests Specific 2.6 # run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation-with-wire-encryption -g storage_formats,cli,hdfs_impersonation,authorization - name: Product Tests Specific 2.7 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-kafka -g kafka - name: Product Tests Specific 2.8 + env: + OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} run: presto-product-tests/bin/run_on_docker.sh singlenode-sqlserver -g sqlserver diff --git a/.github/workflows/singlestore-tests.yml b/.github/workflows/singlestore-tests.yml index 30cc28b08da87..cd4b2c91b7660 100644 --- a/.github/workflows/singlestore-tests.yml +++ b/.github/workflows/singlestore-tests.yml @@ -30,13 +30,17 @@ jobs: codechange: - '!presto-docs/**' singlestore-dockerized-tests: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 30 concurrency: - group: ${{ github.workflow }}-singlestore-dockerized-tests-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-singlestore-dockerized-tests-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -56,7 +60,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/spark-integration.yml b/.github/workflows/spark-integration.yml index b193bfb4a3d03..005b18231c1b1 100644 --- a/.github/workflows/spark-integration.yml +++ b/.github/workflows/spark-integration.yml @@ -30,12 +30,16 @@ jobs: codechange: - '!presto-docs/**' spark-integration: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-spark-integration-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-spark-integration-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -44,7 +48,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/test-other-modules.yml b/.github/workflows/test-other-modules.yml index 4fece0113970a..131a8f4a99ded 100644 --- a/.github/workflows/test-other-modules.yml +++ b/.github/workflows/test-other-modules.yml @@ -30,12 +30,16 @@ jobs: codechange: - '!presto-docs/**' test-other-modules: + strategy: + fail-fast: false + matrix: + java: [ 8, 17.0.13 ] runs-on: ubuntu-latest needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 60 concurrency: - group: ${{ github.workflow }}-test-other-modules-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-test-other-modules-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -44,7 +48,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache run: .github/bin/download_nodejs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5fe34883ad64..84f0d30790739 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,7 @@ jobs: strategy: fail-fast: false matrix: + java: [8, 17.0.13] modules: - ":presto-tests -P presto-tests-execution-memory" - ":presto-tests -P presto-tests-general" @@ -68,7 +69,7 @@ jobs: - ":presto-iceberg" timeout-minutes: 80 concurrency: - group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }}-${{ matrix.java }} cancel-in-progress: true steps: - uses: actions/checkout@v4 @@ -79,7 +80,7 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: distribution: 'temurin' - java-version: 8 + java-version: ${{ matrix.java }} cache: 'maven' - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' diff --git a/.mvn/jvm.config b/.mvn/jvm.config index 608348df8499a..5abda81f42fca 100644 --- a/.mvn/jvm.config +++ b/.mvn/jvm.config @@ -1 +1 @@ --Xmx8192m +-Xmx8192m diff --git a/pom.xml b/pom.xml index a747658ebdb80..a5049e8ec212d 100644 --- a/pom.xml +++ b/pom.xml @@ -2738,6 +2738,46 @@ + + java17 + + 17 + + + + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.lang.ref=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.net=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/java.security=ALL-UNNAMED + --add-opens java.base/javax.security.auth=ALL-UNNAMED + --add-opens java.base/javax.security.auth.login=ALL-UNNAMED + --add-opens java.base/java.text=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens java.base/java.util.regex=ALL-UNNAMED + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED + --add-opens java.base/sun.security.action=ALL-UNNAMED + --add-opens java.base/sun.security.util=ALL-UNNAMED + --add-opens java.base/jdk.internal.module=ALL-UNNAMED + --add-opens java.base/java.lang.module=ALL-UNNAMED + --add-opens java.base/java.util.jar=ALL-UNNAMED + --add-opens java.base/java.util.zip=ALL-UNNAMED + --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED + --add-opens java.base/sun.reflect.annotation=ALL-UNNAMED + --add-opens java.base/sun.nio.fs=ALL-UNNAMED + --add-opens java.base/jdk.internal.ref=ALL-UNNAMED + --add-opens java.base/java.nio.charset=ALL-UNNAMED + --add-opens java.base/java.nio.file.attribute=ALL-UNNAMED + --add-opens java.base/java.lang.invoke=ALL-UNNAMED + --add-opens java.base/sun.net.www.protocol.jar=ALL-UNNAMED + --add-opens java.base/jdk.internal.perf=ALL-UNNAMED + + + ui diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index 60563704da132..48a7b1dd040e2 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -368,5 +368,26 @@ + + skipJdk17IncompatibleTests + + [17,) + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/TestAccumuloDistributedQueries.java + **/TestAccumuloIntegrationSmokeTest.java + **/TestAccumuloClient.java + + + + + + diff --git a/presto-base-arrow-flight/pom.xml b/presto-base-arrow-flight/pom.xml index a670a1a71a881..693f753ee0c0b 100644 --- a/presto-base-arrow-flight/pom.xml +++ b/presto-base-arrow-flight/pom.xml @@ -253,4 +253,18 @@ + + + + java17 + + 17 + + + + --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + diff --git a/presto-druid/pom.xml b/presto-druid/pom.xml index 828be203be15a..c7e8f322a5056 100644 --- a/presto-druid/pom.xml +++ b/presto-druid/pom.xml @@ -130,6 +130,10 @@ it.unimi.dsi fastutil-core + + com.google.inject.extensions + guice-multibindings + diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index ddbf856f7a348..00629aaf8643b 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -630,13 +630,15 @@ test-hive-pushdown-filter-queries-basic + + 5g + org.apache.maven.plugins maven-surefire-plugin - -Xms4g -Xmx4g **/TestHivePushdownFilterQueries.java **/TestHivePushdownIntegrationSmokeTest.java diff --git a/presto-main/etc/jvm.config b/presto-main/etc/jvm.config index 095e663b70c0f..311a1e242079d 100644 --- a/presto-main/etc/jvm.config +++ b/presto-main/etc/jvm.config @@ -4,3 +4,7 @@ # This configuration file is for development only and should NOT be used be # used in production. For example configuration, see the Presto documentation. # + +# Required for Java 17 runtime +#--add-opens=java.base/java.lang=ALL-UNNAMED +#--add-opens=java.base/java.lang.reflect=ALL-UNNAMED diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestKllSketchStateSerializer.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestKllSketchStateSerializer.java index be2e362ee4916..319d56bc94ca7 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestKllSketchStateSerializer.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestKllSketchStateSerializer.java @@ -25,6 +25,7 @@ import org.apache.datasketches.common.ArrayOfStringsSerDe; import org.apache.datasketches.kll.KllItemsSketch; import org.openjdk.jol.info.GraphLayout; +import org.testng.annotations.Ignore; import org.testng.annotations.Test; import java.util.Arrays; @@ -91,6 +92,7 @@ public void testEstimatedMemorySizeLong() testEstimatedMemorySize(BIGINT, i -> (long) i, .05); } + @Ignore("The memory size of a string-typed sketch is dependent upon the strings used as input, so this test doesn't verify much") @Test public void testEstimatedMemorySizeString() { diff --git a/presto-product-tests/conf/docker/common/compose-commons.sh b/presto-product-tests/conf/docker/common/compose-commons.sh index 5a87e5e1bc79a..eae9f18ce9583 100644 --- a/presto-product-tests/conf/docker/common/compose-commons.sh +++ b/presto-product-tests/conf/docker/common/compose-commons.sh @@ -1,6 +1,8 @@ # docker compose has a limited understanding of relative paths and interprets them relative to # compose-file location. We can't guarantee the shape of the paths coming from env variables, # so we canonicalize them. + +set -euo pipefail function export_canonical_path() { # make the path start with '/' or './'. Otherwise the result for 'foo.txt' is an absolute path. local PATH_REFERENCE=$1 @@ -26,6 +28,8 @@ source "${BASH_SOURCE%/*}/../../../bin/locations.sh" export DOCKER_IMAGES_VERSION=${DOCKER_IMAGES_VERSION:-10} export HADOOP_BASE_IMAGE=${HADOOP_BASE_IMAGE:-"prestodb/hdp2.6-hive"} +# This is the directory for the overriden JDK to use for starting Presto +export OVERRIDE_JDK_DIR=${OVERRIDE_JDK_DIR:-"/dev/null"} # The following variables are defined to enable running product tests with arbitrary/downloaded jars # and without building the project. The `presto.env` file should only be sourced if any of the variables diff --git a/presto-product-tests/conf/docker/common/kerberos.yml b/presto-product-tests/conf/docker/common/kerberos.yml index 8737347a9c3eb..144ae610a40e2 100644 --- a/presto-product-tests/conf/docker/common/kerberos.yml +++ b/presto-product-tests/conf/docker/common/kerberos.yml @@ -12,6 +12,8 @@ services: hostname: presto-master image: '${HADOOP_BASE_IMAGE}-kerberized:${DOCKER_IMAGES_VERSION}' command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh singlenode-kerberized run + volumes: + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk networks: default: aliases: diff --git a/presto-product-tests/conf/docker/common/standard.yml b/presto-product-tests/conf/docker/common/standard.yml index 67a1993220fb9..c15ad89190fa3 100644 --- a/presto-product-tests/conf/docker/common/standard.yml +++ b/presto-product-tests/conf/docker/common/standard.yml @@ -39,6 +39,7 @@ services: volumes: - ${PRESTO_SERVER_DIR}:/docker/volumes/presto-server - ../../../src/main/resources/avro:/docker/volumes/presto-product-tests/avro + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk application-runner: extends: diff --git a/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh b/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh index b804cba484611..b4a82d7d4ce1e 100755 --- a/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh +++ b/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -euo pipefail +set -euxo pipefail CONFIG="$1" @@ -14,9 +14,37 @@ if [[ ! -f "${CONFIG_PROPERTIES_LOCATION}" ]]; then exit 1 fi -/docker/volumes/presto-server/bin/launcher \ +JVM_CONFIG="${PRESTO_CONFIG_DIRECTORY}/jvm.config" +JAVA_PROPERTIES="" + +# If we have an overriden JDK volume mount, use it +# This is set to /dev/null ignore +if [ -d /docker/volumes/overridejdk ]; then + CONTAINER_JAVA_HOME=$JAVA_HOME + export JAVA_HOME=/docker/volumes/overridejdk + export PATH=$JAVA_HOME/bin:$PATH + + if [[ "$CONFIG" == "singlenode-ldap" ]]; then + # For LDAP tests use the cacert file from the container JDK which has certs installed + JAVA_PROPERTIES="-Djavax.net.ssl.trustStore=$CONTAINER_JAVA_HOME/jre/lib/security/cacerts" + fi +fi + +echo "Starting Presto with java set to :" +java -version + +# Check if Java version is 17 +# This relies on the version string adhering to the format "17.x.xx" +JAVA_VERSION=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | cut -d'.' -f1) +if [ "$JAVA_VERSION" == "17" ]; then + echo "Java version is 17, setting custom JVM config" + JVM_CONFIG="${PRESTO_CONFIG_DIRECTORY}/jvm17.config" +fi + +/docker/volumes/presto-server/bin/launcher \ -Dnode.id="${HOSTNAME}" \ --etc-dir="${PRESTO_CONFIG_DIRECTORY}" \ + --jvm-config="${JVM_CONFIG}" \ --config="${PRESTO_CONFIG_DIRECTORY}/${CONFIG}".properties \ - --data-dir=/var/presto \ + --data-dir=/var/presto ${JAVA_PROPERTIES} \ "$@" diff --git a/presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml b/presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml index 86e3f3ece9bd3..63e90275d47f3 100644 --- a/presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml +++ b/presto-product-tests/conf/docker/multinode-tls-kerberos/docker-compose.yml @@ -13,6 +13,7 @@ services: - presto-master.docker.cluster volumes: - ../../../conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-no-impersonation/hive.properties:/docker/volumes/conf/presto/etc/catalog/hive.properties + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk presto-worker-1: domainname: docker.cluster diff --git a/presto-product-tests/conf/docker/multinode-tls/docker-compose.yml b/presto-product-tests/conf/docker/multinode-tls/docker-compose.yml index 310feeb09e7f4..505be8a520d55 100644 --- a/presto-product-tests/conf/docker/multinode-tls/docker-compose.yml +++ b/presto-product-tests/conf/docker/multinode-tls/docker-compose.yml @@ -4,6 +4,8 @@ services: domainname: docker.cluster hostname: presto-master command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-tls-master run + volumes: + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk ports: - '7778:7778' networks: diff --git a/presto-product-tests/conf/docker/multinode/docker-compose.yml b/presto-product-tests/conf/docker/multinode/docker-compose.yml index ef2b669b2242d..a7f7ebc2b3422 100644 --- a/presto-product-tests/conf/docker/multinode/docker-compose.yml +++ b/presto-product-tests/conf/docker/multinode/docker-compose.yml @@ -4,6 +4,8 @@ services: command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-master run volumes: - ../../../conf/presto/etc/multinode-master-jvm.config:/docker/volumes/conf/presto/etc/jvm.config + - ../../../conf/presto/etc/multinode-master-jvm17.config:/docker/volumes/conf/presto/etc/jvm17.config + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk presto-worker: extends: @@ -16,3 +18,4 @@ services: - presto-master volumes: - ../../../conf/presto/etc/multinode-worker-jvm.config:/docker/volumes/conf/presto/etc/jvm.config + - ../../../conf/presto/etc/multinode-worker-jvm17.config:/docker/volumes/conf/presto/etc/jvm17.config diff --git a/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml b/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml index 03d15c000819f..b170d7d873dcb 100644 --- a/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml +++ b/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml @@ -9,6 +9,7 @@ services: - '8443:8443' volumes: - ../../../conf/presto/etc/ldap-authenticator.properties:/docker/volumes/conf/presto/etc/password-authenticator.properties + - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk application-runner: image: 'prestodb/centos6-oj8-openldap:${DOCKER_IMAGES_VERSION}' diff --git a/presto-product-tests/conf/presto/etc/catalog/mysql.properties b/presto-product-tests/conf/presto/etc/catalog/mysql.properties index bb6b444c91d09..475c1d6dc1146 100644 --- a/presto-product-tests/conf/presto/etc/catalog/mysql.properties +++ b/presto-product-tests/conf/presto/etc/catalog/mysql.properties @@ -1,4 +1,4 @@ connector.name=mysql -connection-url=jdbc:mysql://mysql:13306 +connection-url=jdbc:mysql://mysql:13306?enabledTLSProtocols=TLSv1.2 connection-user=root connection-password=swarm diff --git a/presto-product-tests/conf/presto/etc/jvm17.config b/presto-product-tests/conf/presto/etc/jvm17.config new file mode 100644 index 0000000000000..d1b67b63931bc --- /dev/null +++ b/presto-product-tests/conf/presto/etc/jvm17.config @@ -0,0 +1,40 @@ +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used be +# used in production. For example configuration, see the Presto documentation. +# + +-server +-Xmx2G +-XX:+UseG1GC +-XX:+UseGCOverheadLimit +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError +-XX:ReservedCodeCacheSize=150M +# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache. +-Djdk.nio.maxCachedBufferSize=0 +-Djdk.attach.allowAttachSelf=true +-DHADOOP_USER_NAME=hive +-Duser.timezone=Asia/Kathmandu +#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 +-XX:ErrorFile=/docker/volumes/logs/product-tests-presto-jvm-error-file.log +# We need to figure out which ones are actually needed, using the same ones as 'air.test.jvm.additional-arguments' +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.lang.ref=ALL-UNNAMED +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED +--add-opens=java.base/java.net=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/java.security=ALL-UNNAMED +--add-opens=java.base/javax.security.auth=ALL-UNNAMED +--add-opens=java.base/javax.security.auth.login=ALL-UNNAMED +--add-opens=java.base/java.text=ALL-UNNAMED +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED +--add-opens=java.base/java.util.regex=ALL-UNNAMED +--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens=java.base/sun.security.action=ALL-UNNAMED +--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \ No newline at end of file diff --git a/presto-product-tests/conf/presto/etc/multinode-master-jvm17.config b/presto-product-tests/conf/presto/etc/multinode-master-jvm17.config new file mode 100644 index 0000000000000..521a7a58cc7e0 --- /dev/null +++ b/presto-product-tests/conf/presto/etc/multinode-master-jvm17.config @@ -0,0 +1,42 @@ +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used be +# used in production. For example configuration, see the Presto documentation. +# + +-server +# coordinator is not a worker +-Xmx1G +-XX:-UseBiasedLocking +# Disable G1 to enable UseGCOverheadLimit (JDK-8212084) and reduce JVM's memory footprint +-XX:+UseParallelGC +-XX:+ExitOnOutOfMemoryError +-XX:+UseGCOverheadLimit +-XX:+HeapDumpOnOutOfMemoryError +-XX:ReservedCodeCacheSize=150M +# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache. +-Djdk.nio.maxCachedBufferSize=0 +-DHADOOP_USER_NAME=hive +-Duser.timezone=Asia/Kathmandu +#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 +-XX:ErrorFile=/docker/volumes/logs/product-tests-presto-jvm-error-file.log +-Djdk.attach.allowAttachSelf=true +# We need to figure out which ones are actually needed, using the same ones as 'air.test.jvm.additional-arguments' +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.lang.ref=ALL-UNNAMED +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED +--add-opens=java.base/java.net=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/java.security=ALL-UNNAMED +--add-opens=java.base/javax.security.auth=ALL-UNNAMED +--add-opens=java.base/javax.security.auth.login=ALL-UNNAMED +--add-opens=java.base/java.text=ALL-UNNAMED +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED +--add-opens=java.base/java.util.regex=ALL-UNNAMED +--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens=java.base/sun.security.action=ALL-UNNAMED +--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED diff --git a/presto-product-tests/conf/presto/etc/multinode-worker-jvm17.config b/presto-product-tests/conf/presto/etc/multinode-worker-jvm17.config new file mode 100644 index 0000000000000..71b7763ab5430 --- /dev/null +++ b/presto-product-tests/conf/presto/etc/multinode-worker-jvm17.config @@ -0,0 +1,42 @@ +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used +# in production. For example configuration, see the Presto documentation. +# + +-server +-Xmx2G +-XX:-UseBiasedLocking +# Disable G1 to enable UseGCOverheadLimit (JDK-8212084) and reduce JVM's memory footprint +-XX:+UseParallelGC +-XX:+ExitOnOutOfMemoryError +-XX:+UseGCOverheadLimit +-XX:+HeapDumpOnOutOfMemoryError +-XX:ReservedCodeCacheSize=150M +-Djdk.attach.allowAttachSelf=true +# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache. +-Djdk.nio.maxCachedBufferSize=0 +-DHADOOP_USER_NAME=hive +-Duser.timezone=Asia/Kathmandu +#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 +-XX:ErrorFile=/docker/volumes/logs/product-tests-presto-jvm-error-file.log +-Djdk.attach.allowAttachSelf=true +# We need to figure out which ones are actually needed, using the same ones as 'air.test.jvm.additional-arguments' +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.lang.ref=ALL-UNNAMED +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED +--add-opens=java.base/java.net=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/java.security=ALL-UNNAMED +--add-opens=java.base/javax.security.auth=ALL-UNNAMED +--add-opens=java.base/javax.security.auth.login=ALL-UNNAMED +--add-opens=java.base/java.text=ALL-UNNAMED +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED +--add-opens=java.base/java.util.regex=ALL-UNNAMED +--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens=java.base/sun.security.action=ALL-UNNAMED +--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index fc7ba21b6a473..e2232426e0eb4 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -18,6 +18,44 @@ 2.12.2 + + + + org.apache.commons + commons-lang3 + 3.14.0 + + + + + + + java17 + + 17 + + + + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.lang.ref=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.net=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/java.security=ALL-UNNAMED + --add-opens java.base/javax.security.auth=ALL-UNNAMED + --add-opens java.base/javax.security.auth.login=ALL-UNNAMED + --add-opens java.base/java.text=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens java.base/java.util.regex=ALL-UNNAMED + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED + --add-opens java.base/sun.security.action=ALL-UNNAMED + + + + org.apache.avro @@ -187,6 +225,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${project.build.targetJdk} + ${project.build.targetJdk} + + org.apache.maven.plugins maven-shade-plugin diff --git a/presto-product-tests/src/main/resources/tempto-configuration.yaml b/presto-product-tests/src/main/resources/tempto-configuration.yaml index 1c9762fec85b6..2b55eeadf5cc4 100644 --- a/presto-product-tests/src/main/resources/tempto-configuration.yaml +++ b/presto-product-tests/src/main/resources/tempto-configuration.yaml @@ -82,7 +82,7 @@ databases: mysql: jdbc_driver_class: com.mysql.jdbc.Driver - jdbc_url: jdbc:mysql://mysql:13306/test + jdbc_url: jdbc:mysql://mysql:13306/test?enabledTLSProtocols=TLSv1.2 jdbc_user: root jdbc_password: swarm jdbc_pooling: true diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 98a02babf58f5..f6e82e02b4cc0 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -13,6 +13,7 @@ ${project.parent.basedir} + 5g @@ -398,7 +399,6 @@ org.apache.maven.plugins maven-surefire-plugin - -Xms4g -Xmx4g **/TestDistributedQueriesNoHashGeneration.java