Merge branch 'develop' into BCF-2612-ChainReader-Next

smartcontractkit · Jan 13, 2024 · d5a0dc6 · d5a0dc6
2 parents 31a7bee + c32efca
commit d5a0dc6
Show file tree

Hide file tree

Showing 330 changed files with 5,463 additions and 3,240 deletions.
diff --git a/.github/tracing/README.md b/.github/tracing/README.md
@@ -48,4 +48,55 @@ This folder contains the following config files:
 
 These config files are for an OTEL collector, grafana Tempo, and a grafana UI instance to run as containers on the same network.
 `otel-collector-dev.yaml` is the configuration for dev (i.e. your local machine) environments, and forwards traces from the otel collector to the grafana tempo instance on the same network. 
-`otel-collector-ci.yaml` is the configuration for the CI runs, and exports the trace data to the artifact from the github run. 
+`otel-collector-ci.yaml` is the configuration for the CI runs, and exports the trace data to the artifact from the github run.
+
+## Adding Traces to Plugins and to core
+
+Adding traces requires identifying an observability gap in a related group of code executions or a critical path in your application. This is intuitive for the developer:
+
+- "What's the flow of component interaction in this distributed system?"
+- "What's the behavior of the JobProcessorOne component when jobs with [x, y, z] attributes are processed?"
+- "Is this critical path workflow behaving the way we expect?"
+
+The developer will measure a flow of execution from end to end in one trace. Each logically separate measure of this flow is called a span. Spans have either one or no parent span and multiple children span. The relationship between parent and child spans in agreggate will form a directed acyclic graph. The trace begins at the root of this graph.
+
+The most trivial application of a span is measuring top level performance in one critical path. There is much more you can do, including creating human readable and timestamped events within a span (useful for monitoring concurrent access to resources), recording errors, linking parent and children spans through large parts of an application, and even extending a span beyond a single process.
+
+Spans are created by `tracers` and passed through go applications by `Context`s. A tracer must be initialized first. Both core and plugin developers will initialize a tracer from the globally registered trace provider:
+
+```
+tracer := otel.GetTracerProvider().Tracer("example.com/foo")
+```
+
+The globally registered tracer provider is available for plugins after they are initialized, and available in core after configuration is processed (`initGlobals`).
+
+Add spans by:
+```
+  func interestingFunc() {
+    // Assuming there is an appropriate parentContext
+	ctx, span := tracer.Start(parentContext, "hello-span")
+	defer span.End()
+
+	// do some work to track with hello-span
+  }
+```
+As implied by the example, `span` is a child of its parent span captured by `parentContext`.
+
+
+Note that in certain situations, there are 3rd party libraries that will setup spans. For instance:
+
+```
+import (
+	"github.com/gin-gonic/gin"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"
+)
+
+router := gin.Default()
+router.Use(otelgin.Middleware("service-name"))
+```
+
+The developer aligns with best practices when they:
+- Start with critical paths
+- Measure paths from end to end (Context is wired all the way through)
+- Emphasize broadness of measurement over depth
+- Use automatic instrumentation if possible
diff --git a/.github/workflows/bash-scripts.yml b/.github/workflows/bash-scripts.yml
@@ -0,0 +1,35 @@
+name: Bash Scripts
+
+on:
+  pull_request:
+
+jobs:
+  changes:
+    name: detect changes
+    runs-on: ubuntu-latest
+    outputs:
+      bash-scripts-src: ${{ steps.bash-scripts.outputs.src }}
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
+        id: bash-scripts
+        with:
+          filters: |
+            src:
+            - 'tools/bin/**'
+            - '.github/workflows/bash-scripts.yml'
+  shellcheck:
+    name: ShellCheck Lint
+    runs-on: ubuntu-latest
+    needs: [changes]
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Run ShellCheck
+        if: needs.changes.outputs.bash-scripts-src == 'true'
+        uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # v2.0.0
+        with:
+          scandir: "./tools/bin"
+          # Consider changing this to check for warnings once all warnings are fixed.
+          severity: error
diff --git a/.github/workflows/build-publish-pr.yml b/.github/workflows/build-publish-pr.yml
@@ -2,8 +2,8 @@ name: "Build and Publish from PR"
 
 ##
 # This workflow builds and publishes a Docker image for Chainlink from a PR.
-# It doesn't use an environment, has its own special IAM role, does not sign
-# the image, and publishes to a special ECR repo.
+# It has its own special IAM role, does not sign the image, and publishes to
+# a special ECR repo.
 ##
 
 on:
@@ -13,6 +13,7 @@ jobs:
   build-publish-untrusted:
     if: ${{ ! startsWith(github.ref_name, 'release/') }}
     runs-on: ubuntu-20.04
+    environment: sdlc
     permissions:
       id-token: write
       contents: read
@@ -53,6 +54,51 @@ jobs:
           dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
           dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
 
+      - name: Get PR labels
+        id: pr-labels
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          RESPONSE=$(gh pr view ${PR_NUMBER} --json labels)
+          # Check if the labels command was successful
+          if [[ $? -ne 0 ]]; then
+            echo "Error fetching labels"
+            exit 1
+          fi
+          echo "RESPONSE=${RESPONSE}"
+          LABELS=$(echo "$RESPONSE" | jq -r '.labels | map(.name) | join(", ")')
+          # Check if any labels were found
+          if [[ -z "${LABELS:-}" ]]; then
+            echo "No labels found"
+          else
+            echo "labels=${LABELS}" | tee -a "${GITHUB_OUTPUT}"
+          fi
+
+      - name: Setup GAP
+        if: contains(steps.pr-labels.outputs.labels, 'crib')
+        uses: smartcontractkit/.github/actions/setup-gap@main
+        with:
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_PUBLISH_PR_ARN }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_ARGO_SAND }}
+          use-argocd: "true"
+          argocd-user: ${{ secrets.ARGOCD_USER_SAND }}
+          argocd-pass: ${{ secrets.ARGOCD_PASS_SAND }}
+
+      # Run an Argo CD sync after the image is built.
+      - name: Argo CD App Sync
+        if: contains(steps.pr-labels.outputs.labels, 'crib')
+        shell: bash
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          argocd app sync \
+            --plaintext \
+            --grpc-web \
+            --async \
+            "crib-chainlink-${PR_NUMBER}"
+
       - name: Collect Metrics
         if: always()
         id: collect-gha-metrics

diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml
@@ -50,7 +50,7 @@ jobs:
       - name: Checkout the repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Setup node
-        uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
+        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
       - name: Setup NodeJS
         uses: ./.github/actions/setup-nodejs
         with:
@@ -97,7 +97,7 @@ jobs:
         working-directory: ./.github/actions/setup-postgres
       - name: Store logs artifacts
         if: always()
-        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: ${{ matrix.cmd }}_logs
           path: |
@@ -136,7 +136,7 @@ jobs:
       - name: Checkout the repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Setup node
-        uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
+        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
       - name: Setup NodeJS
         uses: ./.github/actions/setup-nodejs
         with:
@@ -154,7 +154,7 @@ jobs:
       - name: Setup DB
         run: ./chainlink.test local db preparetest
       - name: Load test outputs
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
           path: ./artifacts
       - name: Build flakey test runner

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -26,7 +26,7 @@ jobs:
 
       - name: Set up Go
         if: ${{ matrix.language == 'go' }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version-file: 'go.mod'
 

diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml
@@ -29,7 +29,7 @@ jobs:
 
       - name: Set up Go
         if: needs.changes.outputs.src == 'true'
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version-file: 'go.mod'
         id: go

diff --git a/.github/workflows/helm-chart-publish.yml b/.github/workflows/helm-chart-publish.yml
@@ -31,7 +31,7 @@ jobs:
         uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
 
       - name: Run chart-releaser
-        uses: helm/chart-releaser-action@be16258da8010256c6e82849661221415f031968 # v1.5.0
+        uses: helm/chart-releaser-action@a917fd15b20e8b64b94d9158ad54cd6345335584 # v1.6.0
         with:
           charts_dir: charts
           config: .github/cr.yaml

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -249,6 +249,16 @@ jobs:
     runs-on: ${{ matrix.product.os }}
     name: ETH Smoke Tests ${{ matrix.product.name }}
     steps:
+      - name: Collect Metrics
+        if: needs.changes.outputs.src == 'true'
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
+        with:
+          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          this-job-name: ETH Smoke Tests ${{ matrix.product.name }}
+          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
+        continue-on-error: true
       - name: Checkout the repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
@@ -285,16 +295,6 @@ jobs:
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
-      - name: Collect Metrics
-        if: always()
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
-        with:
-          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
-          this-job-name: ETH Smoke Tests ${{ matrix.product.name }}
-          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
-        continue-on-error: true
 
   eth-smoke-tests-matrix:
     if: ${{ !(contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') || github.event_name == 'workflow_dispatch') }}
@@ -365,6 +365,16 @@ jobs:
     runs-on: ${{ matrix.product.os }}
     name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
     steps:
+      - name: Collect Metrics
+        if: needs.changes.outputs.src == 'true'
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
+        with:
+          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
+          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
+        continue-on-error: true
       - name: Checkout the repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
@@ -477,23 +487,13 @@ jobs:
         if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
         run: |
           docker logs otel-collector
-      - name: Collect Metrics
-        if: always()
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
-        with:
-          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
-          this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}
-          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
-        continue-on-error: true
       - name: Permissions on traces
         if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
         run: |
           ls -l ./integration-tests/smoke/traces
       - name: Upload Trace Data
         if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: trace-data
           path: ./integration-tests/smoke/traces/trace-data.json

diff --git a/.github/workflows/on-demand-log-poller.yml b/.github/workflows/on-demand-log-poller.yml
@@ -76,7 +76,7 @@ jobs:
         with:
           ref: ${{ env.REF_NAME }}          
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version-file: "integration-tests/go.mod"
           cache: true

diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -15,6 +15,7 @@ on:
           - "CELO_ALFAJORES"
           - "CELO_MAINNET"
           - "BASE_GOERLI"
+          - "BASE_SEPOLIA"
           - "BASE_MAINNET"
           - "BSC_MAINNET"
           - "BSC_TESTNET"

diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml
@@ -31,7 +31,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
       - name: Build and Push
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
         with:
           context: .
           file: core/chainlink.Dockerfile
@@ -72,7 +72,7 @@ jobs:
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
       - name: Publish pprof artifacts
         if: ${{ success() }}
-        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: pprof_results
           path: ./integration-tests/performance/logs

diff --git a/.github/workflows/solidity-hardhat.yml b/.github/workflows/solidity-hardhat.yml
@@ -84,7 +84,7 @@ jobs:
       - name: Rename coverage
         run: mv ./contracts/coverage.json ./contracts/coverage-${{ matrix.split.idx }}.json
       - name: Upload coverage
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: solidity-coverage-${{ matrix.split.idx }}
           path: ./contracts/coverage-${{ matrix.split.idx }}.json
@@ -110,7 +110,7 @@ jobs:
       - name: Make coverage directory
         run: mkdir ./contracts/coverage-reports
       - name: Download coverage
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
           path: ./contracts/coverage-reports
       - name: Display structure of downloaded files

diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
@@ -18,7 +18,7 @@ jobs:
       pull-requests: write
 
     steps:
-    - uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8.0.0
+    - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         exempt-all-pr-assignees: true

diff --git a/.golangci.yml b/.golangci.yml
@@ -10,6 +10,9 @@ linters:
     - misspell
     - rowserrcheck
     - errorlint
+    - unconvert
+    - sqlclosecheck
+    - noctx
 linters-settings:
   exhaustive:
     default-signifies-exhaustive: true