Show E2E test coverage of chainlink node binary (#12835)

* Get E2E test coverage of chainlink node binary * Fix * Fix lint * Add step to upload go coverage folder * Fix * debug * Fix * Fix lint * Debug * Fix * Fix * Show coverage in github workflow console * Fix lint * Cleanup * Add docs * Do not fail if no coverage dir was found * Update show_coverage.go to merge nodes coverage for each test * Cleanup * Update gha * Create NodeCoverageHelper and generate total coverage for all tests * Update covdata artifact name * Fix * Remove unnecessary build step * Update README * Fix * Update readme * Fix * Fixes * Remove ls step from workflow * Fix action version and tag * Bump CTF * Bump CTF
smartcontractkit · Apr 24, 2024 · 9ab3347 · 9ab3347
1 parent 18a9fc8
commit 9ab3347
Show file tree

Hide file tree

Showing 19 changed files with 545 additions and 30 deletions.
diff --git a/.github/actions/build-chainlink-image/action.yml b/.github/actions/build-chainlink-image/action.yml
@@ -33,14 +33,15 @@ runs:
         AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
     - name: Build Image
       if: steps.check-image.outputs.exists != 'true'
-      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.14
       with:
         cl_repo: smartcontractkit/chainlink
         cl_ref: ${{ inputs.git_commit_sha }}
         cl_dockerfile: ${{ inputs.dockerfile }}
         push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ inputs.git_commit_sha }}${{ inputs.tag_suffix }}
         QA_AWS_REGION: ${{ inputs.AWS_REGION }}
         QA_AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
+        GO_COVER_FLAG: true
         dep_evm_sha: ${{ inputs.dep_evm_sha }}
     - name: Print Chainlink Image Built
       shell: sh

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -372,7 +372,7 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -392,6 +392,16 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
+
+      - name: Upload Coverage Data
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: cl-node-coverage-data-${{ matrix.product.name }}
+          path: .covdata
+          retention-days: 1
+
       - name: Print failed test summary
         if: always()
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
@@ -464,7 +474,7 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -484,6 +494,15 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
+
+      - name: Upload Coverage Data
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: cl-node-coverage-data-${{ matrix.product.name }}
+          path: .covdata
+          retention-days: 1          
 
   eth-smoke-tests-matrix:
     if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
@@ -666,7 +685,7 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -686,6 +705,16 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
+
+      - name: Upload Coverage Data
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: cl-node-coverage-data-${{ matrix.product.name }}-${{ matrix.product.tag_suffix }}
+          path: .covdata
+          retention-days: 1
+
       # Run this step when changes that do not need the test to run are made
       - name: Run Setup
         if: needs.changes.outputs.src == 'false'
@@ -778,6 +807,26 @@ jobs:
           this-job-name: Clean up integration environment deployments
         continue-on-error: true
 
+  show-coverage:
+    name: Show Chainlink Node Go Coverage
+    if: always()
+    needs: [cleanup]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        with:
+          repository: smartcontractkit/chainlink
+          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
+      - name: Download All Artifacts
+        uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
+        with:
+          path: cl-node-coverage-data
+          pattern: cl-node-coverage-data-*
+          merge-multiple: true
+      - name: Show Coverage
+        run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged"
+
   # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
   # this will also only run if both of the matrix jobs pass
   eth-smoke-go-mod-cache:
@@ -850,7 +899,7 @@ jobs:
           upgradeImage: ${{ env.UPGRADE_IMAGE }}
           upgradeVersion: ${{ env.UPGRADE_VERSION }}
       - name: Run Migration Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -868,6 +917,15 @@ jobs:
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
+      - name: Upload Coverage Data
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: cl-node-coverage-data-migration-tests
+          path: .covdata
+          retention-days: 1          
+
       - name: Collect Metrics
         if: always()
         id: collect-gha-metrics
@@ -1150,7 +1208,7 @@ jobs:
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11
         with:
           test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
@@ -1168,4 +1226,13 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           run_setup: false
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata          
+
+      - name: Upload Coverage Data
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: cl-node-coverage-data-solana-tests
+          path: .covdata
+          retention-days: 1          
 
diff --git a/.gitignore b/.gitignore
@@ -51,6 +51,7 @@ output.txt
 race.*
 golangci-lint-output.txt
 /golangci-lint/
+.covdata
 
 # DB state
 ./db/

diff --git a/GNUmakefile b/GNUmakefile
@@ -45,6 +45,10 @@ docs: ## Install and run pkgsite to view Go docs
 install-chainlink: operator-ui ## Install the chainlink binary.
 	go install $(GOFLAGS) .
 
+.PHONY: install-chainlink-cover
+install-chainlink-cover: operator-ui ## Install the chainlink binary with cover flag.
+	go install -cover $(GOFLAGS) .
+
 .PHONY: chainlink
 chainlink: ## Build the chainlink binary.
 	go build $(GOFLAGS) .

diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile
@@ -12,12 +12,19 @@ RUN go mod download
 # Env vars needed for chainlink build
 ARG COMMIT_SHA
 
+# Build chainlink bin with cover flag https://go.dev/doc/build-cover#FAQ
+ARG GO_COVER_FLAG=false
+
 COPY . .
 
 RUN apt-get update && apt-get install -y jq
 
 # Build the golang binary
-RUN make install-chainlink
+RUN if [ "$GO_COVER_FLAG" = "true" ]; then \
+        make install-chainlink-cover; \
+    else \
+        make install-chainlink; \
+    fi
 
 # Link LOOP Plugin source dirs with simple names
 RUN go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-feeds | xargs -I % ln -s % /chainlink-feeds
@@ -68,6 +75,11 @@ WORKDIR /home/${CHAINLINK_USER}
 ENV XDG_CACHE_HOME /home/${CHAINLINK_USER}/.cache
 RUN mkdir -p ${XDG_CACHE_HOME}
 
+# Set up env and dir for go coverage profiling https://go.dev/doc/build-cover#FAQ
+ARG GO_COVER_DIR="/var/tmp/go-coverage"
+ENV GOCOVERDIR=${GO_COVER_DIR}
+RUN mkdir -p $GO_COVER_DIR
+
 EXPOSE 6688
 ENTRYPOINT ["chainlink"]
 

diff --git a/integration-tests/docker/README.md b/integration-tests/docker/README.md
@@ -20,3 +20,29 @@ cd ./integration-tests/docker/cmd
 
 go run test_env.go start-env cl-cluster
 ```
+
+## Obtaining Test Coverage for Chainlink Node
+
+To acquire test coverage data for end-to-end (E2E) tests on the Chainlink Node, follow these steps:
+
+1. Build Chainlink Node docker image with the cover flag.
+
+    First, build the Chainlink Node Docker image with the `GO_COVER_FLAG` argument set to `true`. This enables the coverage flag in the build process. Here’s how you can do it:
+    ```
+    docker buildx build --platform linux/arm64 . -t localhost/chainlink-local:develop -f ./core/chainlink.Dockerfile --build-arg GO_COVER_FLAG=true
+    ```
+    Make sure to replace localhost/chainlink-local:develop with the appropriate repository and tag.
+
+2. Configure and Run E2E Tests
+    Next, configure the E2E tests to generate an HTML coverage report. You need to modify the `overrides.toml` file as shown below to include the show_html_coverage_report setting under the `[Logging]` section:
+
+    ```
+    [Logging]
+    show_html_coverage_report=true
+    ```
+
+After the tests are complete, the coverage report will be generated in HTML format. Example: `~/Downloads/go-coverage/TestOCRv2Basic_plugins-chain-reader/coverage.html`
+```
+    log.go:43: 16:29:46.73 INF Chainlink node coverage html report saved filePath=~/Downloads/go-coverage/TestOCRv2Basic_plugins-chain-reader/coverage.html
+```
+