[feat][ci] Collect code coverage for integration tests from docker co…

…ntainers (#19263)

.github/actions/upload-coverage/action.yml

@@ -19,7 +19,8 @@
 
 name: Upload to Codecov with retries
 description: |
-  Uploads to codecov with multiple retries as a workaround 
+  Checks that the current repository is public and then
+  uploads to codecov with multiple retries as a workaround 
   for these issues
     - https://github.com/codecov/codecov-action/issues/598
     - https://github.com/codecov/codecov-action/issues/837
@@ -30,8 +31,26 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: "Check that repository is public"
+      id: repo-check
+      shell: bash
+      run: |
+        if [[ "${{ github.server_url }}" != "https://github.com" ]]; then
+          echo "Not using github.com server ('${{ github.server_url }}'). Skipping uploading of coverage metrics."
+          echo "passed=false" >> $GITHUB_OUTPUT
+          exit 0
+        fi
+        REPO_URL="${{ github.server_url }}/${{ github.repository }}"
+        {
+          # public repository url will respond to http HEAD request
+          curl -X HEAD -fs "$REPO_URL" && echo "passed=true" >> $GITHUB_OUTPUT
+        } || {
+          echo "$REPO_URL isn't a public repository. Skipping uploading of coverage metrics."
+          echo "passed=false" >> $GITHUB_OUTPUT
+        }
     - name: "Upload to Codecov (attempt #1)"
       id: codecov-upload-1
+      if: steps.repo-check.outputs.passed == 'true'
       uses: codecov/codecov-action@v3
       continue-on-error: true
       with:

.github/workflows/pulsar-ci-flaky.yaml

@@ -128,7 +128,6 @@ jobs:
         uses: ./.github/actions/copy-test-reports
 
       - name: Upload to Codecov
-        if: ${{ github.repository == 'apache/pulsar' }}
         uses: ./.github/actions/upload-coverage
         with:
           flags: unittests

.github/workflows/pulsar-ci.yaml

@@ -247,7 +247,6 @@ jobs:
         uses: ./.github/actions/copy-test-reports
 
       - name: Upload to Codecov
-        if: ${{ github.repository == 'apache/pulsar' }}
         uses: ./.github/actions/upload-coverage
         with:
           flags: unittests
@@ -377,6 +376,7 @@ jobs:
         include:
           - name: Backwards Compatibility
             group: BACKWARDS_COMPAT
+            no_coverage: true
 
           - name: Cli
             group: CLI
@@ -388,15 +388,18 @@ jobs:
             group: SHADE_RUN
             runtime_jdk: 8
             setup: ./build/run_integration_group.sh SHADE_BUILD
+            no_coverage: true
 
           - name: Shade on Java 11
             group: SHADE_RUN
             runtime_jdk: 11
             setup: ./build/run_integration_group.sh SHADE_BUILD
+            no_coverage: true
 
           - name: Shade on Java 17
             group: SHADE_RUN
             setup: ./build/run_integration_group.sh SHADE_BUILD
+            no_coverage: true
 
           - name: Standalone
             group: STANDALONE
@@ -467,7 +470,15 @@ jobs:
 
       - name: Run integration test group '${{ matrix.group }}'
         run: |
-          ./build/run_integration_group.sh ${{ matrix.group }}
+          if [[ "${{ matrix.no_coverage }}" != "true" ]]; then
+            coverage_args="--coverage"
+          fi
+          ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args
+
+      - name: Upload to Codecov
+        uses: ./.github/actions/upload-coverage
+        with:
+          flags: inttests
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()
@@ -739,7 +750,12 @@ jobs:
 
       - name: Run system test group '${{ matrix.group }}'
         run: |
-          ./build/run_integration_group.sh ${{ matrix.group }}
+          ./build/run_integration_group.sh ${{ matrix.group }} --coverage
+
+      - name: Upload to Codecov
+        uses: ./.github/actions/upload-coverage
+        with:
+          flags: systests
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()

build/run_integration_group.sh

@@ -27,6 +27,7 @@ set -o errexit
 JAVA_MAJOR_VERSION="$(java -version 2>&1 |grep " version " | awk -F\" '{ print $2 }' | awk -F. '{ if ($1=="1") { print $2 } else { print $1 } }')"
 # Used to shade run test on Java 8, because the latest TestNG requires Java 11 or higher.
 TESTNG_VERSION="7.3.0"
+COVERAGE_COLLECTED=0
 
 # lists all active maven modules with given parameters
 # parses the modules from the "mvn initialize" output
@@ -46,15 +47,14 @@ mvn_list_modules() {
 # 2. runs "mvn -pl [active_modules] -am install [given_params]" to build and install required dependencies
 # 3. finally runs tests with "mvn -pl [active_modules] test [given_params]"
 mvn_run_integration_test() {
-  (
   set +x
   # skip test run if next parameter is "--build-only"
-  build_only=0
+  local build_only=0
   if [[ "$1" == "--build-only" ]]; then
       build_only=1
       shift
   fi
-  skip_build_deps=0
+  local skip_build_deps=0
   while [[ "$1" == "--skip-build-deps" ]]; do
     skip_build_deps=1
     shift
@@ -78,8 +78,19 @@ mvn_run_integration_test() {
   else
     failfast_args="-DtestFailFast=false --fail-at-end"
   fi
+  local coverage_args=""
+  if [[ "$1" == "--coverage" ]]; then
+      if [ ! -d /tmp/jacocoDir ]; then
+          mkdir /tmp/jacocoDir
+          sudo chmod 0777 /tmp/jacocoDir || chmod 0777 /tmp/jacocoDir
+      fi
+      coverage_args="-Pcoverage -Dintegrationtest.coverage.enabled=true -Dintegrationtest.coverage.dir=/tmp/jacocoDir"
+      COVERAGE_COLLECTED=1
+      shift
+  fi
+  (
   cd "$SCRIPT_DIR"/../tests
-  modules=$(mvn_list_modules -DskipDocker "$@")
+  local modules=$(mvn_list_modules -DskipDocker "$@")
   cd ..
   set -x
   if [ $skip_build_deps -ne 1 ]; then
@@ -90,14 +101,46 @@ mvn_run_integration_test() {
   if [[ $build_only -ne 1 ]]; then
     echo "::group::Run tests for " "$@"
     # use "verify" instead of "test"
-    mvn -B -ntp -pl "$modules" $failfast_args -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DredirectTestOutputToFile=false $clean_arg verify "$@"
+    mvn -B -ntp -pl "$modules" $failfast_args $coverage_args -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DredirectTestOutputToFile=false $clean_arg verify "$@"
     echo "::endgroup::"
     set +x
-    "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports
+    "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports || true
   fi
   )
 }
 
+# creates a jacoco xml report of the jacoco exec files produced in docker containers which have /tmp/jacocoDir mounted as /jacocoDir
+# this is used to calculate test coverage for the apache/pulsar code that is run inside the containers in integration tests
+# and system tests
+produce_integration_test_coverage_xml_report() {
+  if [[ -d /tmp/jacocoDir && "$OSTYPE" == "linux-gnu"* && -n "$(find /tmp/jacocoDir -name "*.exec" -print -quit)" ]]; then
+    cd "$GITHUB_WORKSPACE"
+    local jacoco_xml_file=tests/integration/target/site/jacoco/jacoco_inttests.xml
+    echo "Creating coverage report to $jacoco_xml_file"
+    local jacoco_version=$(mvn help:evaluate -Dexpression=jacoco-maven-plugin.version -q -DforceStdout)
+    set -x
+    mkdir -p $(dirname $jacoco_xml_file)
+    # install jacococli.jar command line tool
+    if [ ! -f /tmp/jacocoDir/jacococli.jar ]; then
+      curl -sL -o /tmp/jacocoDir/jacococli.jar "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/${jacoco_version}/org.jacoco.cli-${jacoco_version}-nodeps.jar"
+    fi
+    # extract the Pulsar jar files from the docker image that was used to run the tests in the docker containers
+    # the class files used to produce the jacoco exec files are needed in the xml report generation
+    if [ ! -d /tmp/jacocoDir/pulsar_lib ]; then
+      mkdir /tmp/jacocoDir/pulsar_lib
+      docker run --rm -u "$UID:${GID:-"$(id -g)"}" -v /tmp/jacocoDir/pulsar_lib:/pulsar_lib:rw ${PULSAR_TEST_IMAGE_NAME:-apachepulsar/java-test-image:latest} bash -c "cp -p /pulsar/lib/org.apache.pulsar-* /pulsar_lib"
+      # remove jar file that causes duplicate classes issue
+      rm /tmp/jacocoDir/pulsar_lib/org.apache.pulsar-bouncy-castle*
+    fi
+    # produce jacoco XML coverage report from the exec files and using the extracted jar files
+    java -jar /tmp/jacocoDir/jacococli.jar report /tmp/jacocoDir/*.exec \
+      --classfiles /tmp/jacocoDir/pulsar_lib --encoding UTF-8 --name "Pulsar Integration Tests - coverage in containers" \
+      $(find -path "*/src/main/java" -printf "--sourcefiles %P ") \
+      --xml $jacoco_xml_file
+    set +x
+  fi
+}
+
 test_group_shade() {
   mvn_run_integration_test "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false
 }
@@ -223,6 +266,9 @@ echo "Test Group : $TEST_GROUP"
 test_group_function_name="test_group_$(echo "$TEST_GROUP" | tr '[:upper:]' '[:lower:]')"
 if [[ "$(LC_ALL=C type -t "${test_group_function_name}")" == "function" ]]; then
   eval "$test_group_function_name" "$@"
+  if [ $COVERAGE_COLLECTED -eq 1 ]; then
+      produce_integration_test_coverage_xml_report
+  fi
 else
   echo "INVALID TEST GROUP"
   echo "Available test groups:"

codecov.yml

@@ -21,12 +21,17 @@ codecov:
   require_ci_to_pass: yes
   notify:
     # should match the number of builds sending coverage reports
-    # currently pulsar-ci.yaml contains 9 builds and pulsar-ci-flaky.yaml contains 1 build
-    after_n_builds: 10
+    # pulsar-ci.yaml contains:
+    #   - 9 unit test builds
+    #   - 4 integration test builds (without 'no_coverage: true')
+    #   - 8 system test build
+    # pulsar-ci-flaky.yaml contains:
+    #   - 1 build
+    after_n_builds: 22
 
 comment:
   # should match the number of builds sending coverage reports
-  after_n_builds: 10
+  after_n_builds: 22
   layout: "reach, diff, flags, files"
   behavior: default
   require_changes: false

pom.xml

@@ -111,6 +111,8 @@ flexible messaging model and an intuitive client API.</description>
     <testFailFast>true</testFailFast>
     <testFailFastFile></testFailFastFile>
     <testJacocoAgentArgument/>
+    <integrationtest.coverage.enabled>false</integrationtest.coverage.enabled>
+    <integrationtest.coverage.dir>${project.build.directory}</integrationtest.coverage.dir>
     <testHeapDumpPath>/tmp</testHeapDumpPath>
     <surefire.shutdown>kill</surefire.shutdown>
     <docker.organization>apachepulsar</docker.organization>

tests/docker-images/latest-version-image/conf/bookie.conf

@@ -25,3 +25,4 @@ directory=/pulsar
 environment=PULSAR_MEM="-Xmx128M -XX:MaxDirectMemorySize=512M",PULSAR_GC="-XX:+UseZGC"
 command=/pulsar/bin/pulsar bookie
 user=pulsar
+stopwaitsecs=15

tests/docker-images/latest-version-image/conf/broker.conf

@@ -25,3 +25,4 @@ directory=/pulsar
 environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
 command=/pulsar/bin/pulsar broker
 user=pulsar
+stopwaitsecs=15

tests/docker-images/latest-version-image/conf/functions_worker.conf

@@ -25,3 +25,4 @@ directory=/pulsar
 environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
 command=/pulsar/bin/pulsar functions-worker
 user=pulsar
+stopwaitsecs=15

tests/docker-images/latest-version-image/conf/global-zk.conf

@@ -25,3 +25,4 @@ directory=/pulsar
 environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
 command=/pulsar/bin/pulsar configuration-store
 user=pulsar
+stopwaitsecs=15

tests/docker-images/latest-version-image/conf/local-zk.conf

@@ -25,3 +25,4 @@ directory=/pulsar
 environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
 command=/pulsar/bin/pulsar zookeeper
 user=pulsar
+stopwaitsecs=15

tests/docker-images/latest-version-image/conf/presto_worker.conf

@@ -25,3 +25,4 @@ directory=/pulsar
 environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
 command=/pulsar/bin/pulsar sql-worker start
 user=pulsar
+stopwaitsecs=15

tests/docker-images/latest-version-image/conf/proxy.conf

@@ -25,3 +25,4 @@ directory=/pulsar
 environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
 command=/pulsar/bin/pulsar proxy
 user=pulsar
+stopwaitsecs=15

tests/docker-images/latest-version-image/conf/websocket.conf

@@ -25,3 +25,4 @@ directory=/pulsar
 environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
 command=/pulsar/bin/pulsar websocket
 user=pulsar
+stopwaitsecs=15

tests/integration/pom.xml

@@ -280,7 +280,8 @@
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
               <argLine>${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx1G -XX:MaxDirectMemorySize=1G
-              -Dio.netty.leakDetectionLevel=advanced -Dconfluent.version=${confluent.version}
+              -Dio.netty.leakDetectionLevel=advanced -Dconfluent.version=${confluent.version} -Djacoco.version=${jacoco-maven-plugin.version}
+              -Dintegrationtest.coverage.enabled=${integrationtest.coverage.enabled} -Dintegrationtest.coverage.dir=${integrationtest.coverage.dir}
               ${test.additional.args}
               </argLine>
               <skipTests>false</skipTests>

tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java

@@ -63,6 +63,10 @@ protected void beforeStop() {
     @Override
     public void stop() {
         beforeStop();
+        doStop();
+    }
+
+    protected void doStop() {
         super.stop();
     }