From a742eb53cfc5aaaaf5896be04de11df5d5e0ef40 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Mon, 9 May 2022 01:52:54 -0700 Subject: [PATCH] [improve][build] PIP-156 Build Pulsar Server on Java 17 (#15264) * [PIP-156][Build] minimal build change to set javac release to 17 except client (shared) modules * [PIP-156][fix][ci] fixed failed tests due to the java 17 build change * [PIP-156][fix][build] removed unused pulsar-transaction-common dependency from pulsar-client * [PIP-156][Build] build additional common modules(shared to pulsar-client) to target Java 8 * [PIP-156][ci] updated pulsar-client-cpp docker image with pre-built jdk17 * [PIP-156][ci] added CI integration-shade-test for pulsar-client's different java runtime compatibility * [PIP-156][doc] updated docs as per java 17 build udpate * [PIP-156][ci] trying to fix the yaml format error from pulsar-ci.yaml * [PIP-156][ci] fixed the Shade CI (java11) build error * [PIP-156][build] resolved comments: mvn property name change and doc update * [PIP-156][doc] resolved more comments in the docs * Revert changes to pulsar-ci.yaml * Extend existing integration-tests job to support shade tests on multiple JDKs * Revert "[PIP-156][ci] updated pulsar-client-cpp docker image with pre-built jdk17" This reverts commit 7d710783357df85c656b546949f47c215ded70c4. * Revert changes to .asf.yaml Co-authored-by: Lari Hotari --- .github/workflows/pulsar-ci.yaml | 25 +++++++++--- README.md | 38 ++++++++++++++++--- bouncy-castle/pom.xml | 14 +++++++ build/run_integration_group.sh | 28 +++++++++++--- build/run_unit_group.sh | 4 -- pom.xml | 5 ++- .../service/ManagedLedgerCompressionTest.java | 2 +- pulsar-client-1x-base/pom.xml | 7 ++++ pulsar-client-admin-api/pom.xml | 7 ++++ pulsar-client-admin/pom.xml | 7 ++++ pulsar-client-all/pom.xml | 1 - pulsar-client-api/pom.xml | 7 ++++ pulsar-client-cpp/docker-tests.sh | 5 ++- pulsar-client-messagecrypto-bc/pom.xml | 7 ++++ pulsar-client-shaded/pom.xml | 1 - pulsar-client/pom.xml | 13 ++++--- pulsar-common/pom.xml | 8 ++++ site2/docs/deploy-bare-metal.md | 2 +- site2/docs/functions-package.md | 7 +++- site2/docs/io-develop.md | 2 + 20 files changed, 155 insertions(+), 35 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index a3d0c4bed989a..7fd807a3b99a7 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -150,9 +150,6 @@ jobs: group: BROKER_GROUP_2 - name: Brokers - Broker Group 3 group: BROKER_GROUP_3 - - name: Brokers - JDK 8 - group: BROKER_JDK8 - jdk: 8 - name: Brokers - Client Api group: BROKER_CLIENT_API - name: Brokers - Client Impl @@ -347,8 +344,19 @@ jobs: - name: Messaging group: MESSAGING - - name: Shade - group: SHADE + - name: Shade on Java 8 + group: SHADE_RUN + runtime_jdk: 8 + setup: ./build/run_integration_group.sh SHADE_BUILD + + - name: Shade on Java 11 + group: SHADE_RUN + runtime_jdk: 11 + setup: ./build/run_integration_group.sh SHADE_BUILD + + - name: Shade on Java 17 + group: SHADE_RUN + setup: ./build/run_integration_group.sh SHADE_BUILD - name: Standalone group: STANDALONE @@ -410,6 +418,13 @@ jobs: run: | ${{ matrix.setup }} + - name: Set up runtime JDK ${{ matrix.runtime_jdk }} + uses: actions/setup-java@v2 + if: ${{ matrix.runtime_jdk && needs.changed_files_job.outputs.docs_only != 'true' }} + with: + distribution: 'temurin' + java-version: ${{ matrix.runtime_jdk }} + - name: Run integration test group '${{ matrix.group }}' if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} run: | diff --git a/README.md b/README.md index fa4c03d214d37..386b04e875daf 100644 --- a/README.md +++ b/README.md @@ -83,10 +83,37 @@ components in the Pulsar ecosystem, including connectors, adapters, and other la - [Pulsar CI](https://github.com/apache/pulsar-test-infra) +## Pulsar Runtime Java Version Recommendation + +- pulsar master branch + +| Pulsar Components | Java Version | +| ----------------- |:-------------:| +| Broker | 17 | +| Functions / IO | 17 | +| CLI | 17 | +| Java Client | 8 or 11 or 17 | + +- 2.8 <= pulsar ver <= 2.10 + +| Pulsar Components | Java Version | +| ----------------- |:-------------:| +| Broker | 11 | +| Functions / IO | 11 | +| CLI | 8 or 11 | +| Java Client | 8 or 11 | + +- pulsar ver 2.8 < + +| Pulsar Components | Java Version | +| ----------------- |:-------------:| +| All | 8 or 11 | + + ## Build Pulsar Requirements: - * Java [JDK 11](https://adoptium.net/?variant=openjdk11) or [JDK 8](https://adoptium.net/?variant=openjdk8) + * Java [JDK 17](https://adoptium.net/?variant=openjdk17) * Maven 3.6.1+ * zip @@ -136,7 +163,8 @@ Check https://pulsar.apache.org for documentation and examples. Docker images must be built with Java 8 for `branch-2.7` or previous branches because of [issue 8445](https://github.com/apache/pulsar/issues/8445). -Java 11 is the recommended JDK version in `master`/`branch-2.8`. +Java 11 is the recommended JDK version in `branch-2.8`, `branch-2.9` and `branch-2.10`. +Java 17 is the recommended JDK version in `master`. This builds the docker images `apachepulsar/pulsar-all:latest` and `apachepulsar/pulsar:latest`. @@ -168,7 +196,7 @@ required plugins. ### Intellij -#### Configure Project JDK to Java 11 JDK +#### Configure Project JDK to Java 17 JDK 1. Open **Project Settings**. @@ -176,9 +204,9 @@ required plugins. 2. Select the JDK version. - From the JDK version drop-down list, select **Download JDK...** or choose an existing recent Java 11 JDK version. + From the JDK version drop-down list, select **Download JDK...** or choose an existing recent Java 17 JDK version. -3. In the download dialog, select version **11**. You can pick a version from many vendors. Unless you have a specific preference, choose **Eclipse Temurin (AdoptOpenJDK (Hotspot))**. +3. In the download dialog, select version **17**. You can pick a version from many vendors. Unless you have a specific preference, choose **Eclipse Temurin (AdoptOpenJDK (Hotspot))**. #### Configure Java version for Maven in IntelliJ diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 9a946e290460c..3ead292e4b714 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -29,6 +29,20 @@ .. + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + + + + bouncy-castle-parent Apache Pulsar :: Bouncy Castle :: Parent diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh index 630a08ea8367f..bb854a6f28048 100755 --- a/build/run_integration_group.sh +++ b/build/run_integration_group.sh @@ -55,6 +55,12 @@ mvn_run_integration_test() { RETRY="" shift fi + # skip test run if next parameter is "--build-only" + build_only=0 + if [[ "$1" == "--build-only" ]]; then + build_only=1 + shift + fi skip_build_deps=0 while [[ "$1" == "--skip-build-deps" ]]; do skip_build_deps=1 @@ -69,12 +75,14 @@ mvn_run_integration_test() { mvn -B -T 1C -ntp -pl "$modules" -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dmaven.test.skip=true -Dcheckstyle.skip=true -Drat.skip=true -am install "$@" echo "::endgroup::" fi - echo "::group::Run tests for " "$@" - # use "verify" instead of "test" - $RETRY mvn -B -ntp -pl "$modules" -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DredirectTestOutputToFile=false verify "$@" - echo "::endgroup::" - set +x - "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports + if [[ $build_only -ne 1 ]]; then + echo "::group::Run tests for " "$@" + # use "verify" instead of "test" + $RETRY mvn -B -ntp -pl "$modules" -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DredirectTestOutputToFile=false verify "$@" + echo "::endgroup::" + set +x + "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports + fi ) } @@ -82,6 +90,14 @@ test_group_shade() { mvn_run_integration_test "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false } +test_group_shade_build() { + mvn_run_integration_test --build-only "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false +} + +test_group_shade_run() { + mvn_run_integration_test --skip-build-deps "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false -Dmaven.compiler.source=8 -Dmaven.compiler.target=8 +} + test_group_backwards_compat() { mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-backwards-compatibility.xml -DintegrationTests mvn_run_integration_test --retry "$@" -DBackwardsCompatTests -DtestForkCount=1 -DtestReuseFork=false diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 3414a5fb141e8..2431c6305d89a 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -78,10 +78,6 @@ function test_group_broker_client_impl() { mvn_test -pl pulsar-broker -Dgroups='broker-impl' } -function test_group_broker_jdk8() { - mvn_test -pl pulsar-broker -Dgroups='broker-jdk8' -Dpulsar.allocator.pooled=true -} - # prints summaries of failed tests to console # by using the targer/surefire-reports files # works only when testForkCount > 1 since that is when surefire will create reports for individual test classes diff --git a/pom.xml b/pom.xml index bb44a774496e7..b1e3c67e7d045 100644 --- a/pom.xml +++ b/pom.xml @@ -77,8 +77,9 @@ flexible messaging model and an intuitive client API. - 8 - 8 + 17 + 17 + 8 **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java index 3a2390be929a6..30b4aca22bc9b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java @@ -34,7 +34,7 @@ * ManagedLedgerInfo compression configuration test. */ -@Test(groups = {"broker", "broker-jdk8"}) +@Test(groups = {"broker"}) public class ManagedLedgerCompressionTest extends BrokerTestBase { @BeforeClass diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 609403de279de..ea5503eebf540 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -40,6 +40,13 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + com.github.spotbugs spotbugs-maven-plugin diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index ba90a907a3fd5..5083efc41fe0e 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -47,6 +47,13 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + org.gaul modernizer-maven-plugin diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 4eef835b58d90..031590bd3c9d1 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -116,6 +116,13 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + org.gaul modernizer-maven-plugin diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index fb50b96ca6243..6645488d70471 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -134,7 +134,6 @@ org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-transaction-common org.apache.pulsar:pulsar-client-admin-original org.apache.commons:commons-lang3 commons-codec:commons-codec diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index e8c9574058387..d7ebace911eac 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -51,6 +51,13 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + org.gaul modernizer-maven-plugin diff --git a/pulsar-client-cpp/docker-tests.sh b/pulsar-client-cpp/docker-tests.sh index e0296fdd3072d..1665aa86e2507 100755 --- a/pulsar-client-cpp/docker-tests.sh +++ b/pulsar-client-cpp/docker-tests.sh @@ -65,7 +65,10 @@ DISABLE_COLOR_OUTPUT="" if [ "$GTEST_COLOR" = "no" ]; then DISABLE_COLOR_OUTPUT="| cat" fi -$DOCKER_CMD bash -c "set -o pipefail; cd /pulsar/pulsar-client-cpp && ./run-unit-tests.sh ${tests} $DISABLE_COLOR_OUTPUT" + +# Java17 is required for CLI e.g) bin/pulsar create-token +$DOCKER_CMD bash -c "apt-get -y install openjdk-17-jre-headless &&\ + set -o pipefail; cd /pulsar/pulsar-client-cpp && ./run-unit-tests.sh ${tests} $DISABLE_COLOR_OUTPUT" RES=$? if [ $RES -ne 0 ]; then ( diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index f872817fe8c83..e5b8fc56d4691 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -53,6 +53,13 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index c2f6b6755815b..87957c719e4d0 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -129,7 +129,6 @@ org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-transaction-common org.apache.bookkeeper:* org.apache.commons:commons-lang3 commons-codec:commons-codec diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 1c220e415393f..5be8fcd439aaf 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -45,12 +45,6 @@ ${project.parent.version} - - ${project.groupId} - pulsar-transaction-common - ${project.parent.version} - - ${project.groupId} bouncy-castle-bc @@ -195,6 +189,13 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + org.gaul modernizer-maven-plugin diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 954ba986b2ac9..3fdd470cb2472 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -210,6 +210,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + org.gaul modernizer-maven-plugin diff --git a/site2/docs/deploy-bare-metal.md b/site2/docs/deploy-bare-metal.md index 6d12f2dfae5c1..821f1b80feb0a 100644 --- a/site2/docs/deploy-bare-metal.md +++ b/site2/docs/deploy-bare-metal.md @@ -40,7 +40,7 @@ To run Pulsar on bare metal, the following configuration is recommended: > * Broker is only supported on 64-bit JVM. > * If you do not have enough machines, or you want to test Pulsar in cluster mode (and expand the cluster later), You can fully deploy Pulsar on a node on which ZooKeeper, bookie and broker run. > * If you do not have a DNS server, you can use the multi-host format in the service URL instead. -Each machine in your cluster needs to have [Java 8](https://adoptium.net/?variant=openjdk8), [Java 11](https://adoptium.net/?variant=openjdk11) or [Java 17](https://adoptium.net/?variant=openjdk17) installed. +> * Each machine in your cluster needs to have the recommended Java version installed (e.g. [Java 17](https://adoptium.net/?variant=openjdk17)). Please refer to [Pulsar Runtime Java Version Recommendation](https://github.com/apache/pulsar/blob/master/README.md#pulsar-runtime-java-version-recommendation) according to your target Pulsar version. The following is a diagram showing the basic setup: diff --git a/site2/docs/functions-package.md b/site2/docs/functions-package.md index 6158eba2ed400..dbbeffa6e6cb3 100644 --- a/site2/docs/functions-package.md +++ b/site2/docs/functions-package.md @@ -17,6 +17,10 @@ To check whether the Docker image starts, you can use the `docker ps` command. ## Java +> **Note** +> +> For the runtime Java version, please refer to [Pulsar Runtime Java Version Recommendation](https://github.com/apache/pulsar/blob/master/README.md#pulsar-runtime-java-version-recommendation) according to your target Pulsar version. + To package a function in Java, complete the following steps. 1. Create a new maven project with a pom file. In the following code sample, the value of `mainClass` is your package name. @@ -69,8 +73,7 @@ To package a function in Java, complete the following steps. org.apache.maven.plugins maven-compiler-plugin - 8 - 8 + 17 diff --git a/site2/docs/io-develop.md b/site2/docs/io-develop.md index 3c5af1873b192..4dff809e2b534 100644 --- a/site2/docs/io-develop.md +++ b/site2/docs/io-develop.md @@ -261,6 +261,8 @@ all libraries your code uses and to your distribution. > If you use the [NAR](#nar) method, the NAR plugin automatically creates a `DEPENDENCIES` file in the generated NAR package, including the proper licensing and copyrights of all libraries of your connector. +> +> For the runtime Java version, please refer to [Pulsar Runtime Java Version Recommendation](https://github.com/apache/pulsar/blob/master/README.md#pulsar-runtime-java-version-recommendation) according to your target Pulsar version. ### NAR