fix(ci): run static analysis reporting only once per version COMPASS-7998

.evergreen.yml

@@ -1,4 +1,3 @@
-unset_function_vars: true
 stepback: false
 exec_timeout_secs: 5400
 ignore:

.evergreen/buildvariants.in.yml

@@ -18,6 +18,7 @@ buildvariants:
       - name: publish
       - name: publish-packages-next
       - name: publish-dev-release-info
+      - name: create_static_analysis_report
 
   - name: ubuntu_connectivity_tests
     display_name: Connectivity Tests
@@ -36,4 +37,4 @@ buildvariants:
     run_on: <% out(buildVariant.run_on) %>
     tasks:<% for (const task of buildVariant.tasks) { %>
       - name: <% out(task.name); task.run_on && out(`\n        run_on: ${task.run_on}`); task.depends_on && out(`\n        depends_on: ${task.depends_on}`); %>
-<% }} %>
+<% }} %>

.evergreen/buildvariants.yml

@@ -18,6 +18,7 @@ buildvariants:
       - name: publish
       - name: publish-packages-next
       - name: publish-dev-release-info
+      - name: create_static_analysis_report
 
   - name: ubuntu_connectivity_tests
     display_name: Connectivity Tests
@@ -301,3 +302,4 @@ buildvariants:
       - name: test-packaged-app-80x-enterprise
         run_on: macos-1100-arm64-gui
         depends_on: package-compass
+

.evergreen/create-static-analysis-report.sh

@@ -2,5 +2,5 @@
 set -e
 set -x
 
-npm run create-static-analysis-report
+npm run create-static-analysis-report -- --first-party-deps-list-files="${FIRST_PARTY_DEPENDENCY_FILENAMES}"
 (cd .sbom && tar czvf ../static-analysis-report.tgz codeql.md codeql.sarif.json)

.evergreen/functions.yml

@@ -331,7 +331,6 @@ functions:
         wait: true
     # copy ssh key (this key will be used to connect to ssh server)
     - command: shell.exec
-      silent: true
       params:
         working_dir: src
         shell: bash
@@ -429,6 +428,14 @@ functions:
           set -e
 
           .evergreen/create-sbom.sh
+
+  get_first_party_dependency_list:
+    - command: s3.get
+      params:
+        <<: *get-artifact-params
+        local_file: src/.sbom/${filename}
+        remote_file: ${project}/${revision}_${revision_order_id}/${filename}
+  create_static_analysis_report:
     - command: shell.exec
       params:
         working_dir: src
@@ -439,16 +446,18 @@ functions:
           npm_config_loglevel: ${npm_loglevel}
           GITHUB_TOKEN: ${devtoolsbot_github_token}
           GITHUB_PR_NUMBER: ${github_pr_number}
+          FIRST_PARTY_DEPENDENCY_FILENAMES: ${first_party_dependency_filenames}
         script: |
           set -e
           # Load environment variables
           eval $(.evergreen/print-compass-env.sh)
-
-          # Do not run static analysis report generation on patches to reduce
-          # the load on GitHub API that is used in the process
-          if [[ "$EVERGREEN_IS_PATCH" != "true" ]]; then
-            .evergreen/create-static-analysis-report.sh
-          fi
+          .evergreen/create-static-analysis-report.sh 
+    - command: s3.put
+      params:
+        <<: *save-artifact-params-private
+        local_file: src/static-analysis-report.tgz
+        remote_file: ${project}/${revision}_${revision_order_id}/static-analysis-report.tgz
+        content_type: application/x-gzip
 
   publish:
     - command: shell.exec
@@ -807,10 +816,10 @@ functions:
         optional: true
     - command: s3.put
       params:
-        <<: *save-artifact-params-private
-        local_file: src/static-analysis-report.tgz
-        remote_file: ${project}/${revision}_${revision_order_id}/${task_id}/static-analysis-report.tgz
-        content_type: application/x-gzip
+        <<: *save-artifact-params-public
+        local_file: src/.sbom/first-party-deps.json
+        remote_file: ${project}/${revision}_${revision_order_id}/first-party-deps-${compass_distribution}-${build_variant}.json
+        content_type: application/json
         optional: true
 
   get-all-artifacts:

.evergreen/generative-ai-accuracy-test.yml

@@ -2,7 +2,6 @@
 # We don't want it to run on every commit as that would be
 # too many requests to our ai model (expensive).
 
-unset_function_vars: true
 stepback: false
 exec_timeout_secs: 5400
 ignore:

.evergreen/tasks.in.yml

@@ -103,7 +103,7 @@ tasks:
       - func: publish-dev-release-info
 <% for (const packageTask of tasks.package) { %>
   - name: <% out(packageTask.name) %>
-    tags: ['required-for-publish', 'run-on-pr']
+    tags: ['required-for-publish', 'run-on-pr', 'package-task']
     commands:
       - func: prepare
       - func: install
@@ -161,4 +161,29 @@ tasks:
           <% out(`${key}: '${value}'`) } %>
           compass_distribution: compass
           debug: 'compass-e2e-tests*,electron*,hadron*,mongo*'
-<% } %>
+<% } %>
+  - name: create_static_analysis_report
+    tags: ['required-for-publish', 'run-on-pr']
+    depends_on:
+      - name: ".package-task"
+        variant: "*"
+    commands:
+      - func: prepare
+      - func: install
+<%
+let firstPartyDepsFilenames = [];
+for (const packageTask of tasks.package) { 
+  const filenames = [
+    `first-party-deps-${packageTask.vars.compass_distribution}-ubuntu.json`,
+    `first-party-deps-${packageTask.vars.compass_distribution}-windows.json`,
+    `first-party-deps-${packageTask.vars.compass_distribution}-macos.json`,
+    `first-party-deps-${packageTask.vars.compass_distribution}-macos-arm.json`];
+  for (const filename of filenames) {
+  firstPartyDepsFilenames.push(filename); %>
+      - func: get_first_party_dependency_list
+        vars:
+          filename: <% out(filename) %>
+<% } } %>
+      - func: create_static_analysis_report
+        vars:
+          first_party_dependency_filenames: <% out(firstPartyDepsFilenames.map(f => `.sbom/${f}`).join(',')) %>

.evergreen/tasks.yml

@@ -103,7 +103,7 @@ tasks:
       - func: publish-dev-release-info
 
   - name: package-compass
-    tags: ['required-for-publish', 'run-on-pr']
+    tags: ['required-for-publish', 'run-on-pr', 'package-task']
     commands:
       - func: prepare
       - func: install
@@ -124,7 +124,7 @@ tasks:
           compass_distribution: compass
 
   - name: package-compass-readonly
-    tags: ['required-for-publish', 'run-on-pr']
+    tags: ['required-for-publish', 'run-on-pr', 'package-task']
     commands:
       - func: prepare
       - func: install
@@ -145,7 +145,7 @@ tasks:
           compass_distribution: compass-readonly
 
   - name: package-compass-isolated
-    tags: ['required-for-publish', 'run-on-pr']
+    tags: ['required-for-publish', 'run-on-pr', 'package-task']
     commands:
       - func: prepare
       - func: install
@@ -502,3 +502,64 @@ tasks:
           browser_name: 'firefox'
           compass_distribution: compass
           debug: 'compass-e2e-tests*,electron*,hadron*,mongo*'
+
+  - name: create_static_analysis_report
+    tags: ['required-for-publish', 'run-on-pr']
+    depends_on:
+      - name: ".package-task"
+        variant: "*"
+    commands:
+      - func: prepare
+      - func: install
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-ubuntu.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-windows.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-macos.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-macos-arm.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-readonly-ubuntu.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-readonly-windows.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-readonly-macos.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-readonly-macos-arm.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-isolated-ubuntu.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-isolated-windows.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-isolated-macos.json
+
+      - func: get_first_party_dependency_list
+        vars:
+          filename: first-party-deps-compass-isolated-macos-arm.json
+
+      - func: create_static_analysis_report
+        vars:
+          first_party_dependency_filenames: .sbom/first-party-deps-compass-ubuntu.json,.sbom/first-party-deps-compass-windows.json,.sbom/first-party-deps-compass-macos.json,.sbom/first-party-deps-compass-macos-arm.json,.sbom/first-party-deps-compass-readonly-ubuntu.json,.sbom/first-party-deps-compass-readonly-windows.json,.sbom/first-party-deps-compass-readonly-macos.json,.sbom/first-party-deps-compass-readonly-macos-arm.json,.sbom/first-party-deps-compass-isolated-ubuntu.json,.sbom/first-party-deps-compass-isolated-windows.json,.sbom/first-party-deps-compass-isolated-macos.json,.sbom/first-party-deps-compass-isolated-macos-arm.json

package.json

@@ -52,8 +52,8 @@
     "pregenerate-vulnerability-report": "npm run compile -w packages/compass && npm run snyk-test",
     "generate-vulnerability-report": "mongodb-sbom-tools generate-vulnerability-report --snyk-reports=.sbom/snyk-test-result.json --dependencies=.sbom/dependencies.json --fail-on=high > .sbom/vulnerability-report.md",
     "create-vulnerability-tickets": "mongodb-sbom-tools generate-vulnerability-report --snyk-reports=.sbom/snyk-test-result.json --dependencies=.sbom/dependencies.json --create-jira-issues",
-    "precreate-static-analysis-report": "npm run compile -w packages/compass",
-    "create-static-analysis-report": "mongodb-sbom-tools fetch-codeql-results --sarif-dest=.sbom/codeql.sarif.json --dependencies=.sbom/dependencies.json --exclude-repos=mongodb-js/kerberos",
+    "postgenerate-vulnerability-report": "mongodb-sbom-tools fetch-codeql-results --first-party-deps-list-dest=.sbom/first-party-deps.json --dependencies=.sbom/dependencies.json --exclude-repos=mongodb-js/kerberos",
+    "create-static-analysis-report": "mongodb-sbom-tools fetch-codeql-results --sarif-dest=.sbom/codeql.sarif.json",
     "postcreate-static-analysis-report": "mongodb-sbom-tools sarif-to-markdown --sarif=.sbom/codeql.sarif.json --md=.sbom/codeql.md",
     "precommit": "precommit"
   },
@@ -66,7 +66,7 @@
     "@babel/core": "7.16.0",
     "@babel/parser": "7.16.0",
     "@mongodb-js/monorepo-tools": "^1.1.1",
-    "@mongodb-js/sbom-tools": "^0.6.2",
+    "@mongodb-js/sbom-tools": "^0.7.0",
     "@testing-library/dom": "^8.20.1",
     "babel-loader": "^7.1.5",
     "husky": "^8.0.3",

packages/compass/package.json

@@ -227,7 +227,7 @@
     "@mongodb-js/mongodb-downloader": "^0.3.0",
     "@mongodb-js/my-queries-storage": "^0.8.1",
     "@mongodb-js/prettier-config-compass": "^1.0.2",
-    "@mongodb-js/sbom-tools": "^0.6.2",
+    "@mongodb-js/sbom-tools": "^0.7.0",
     "@mongodb-js/tsconfig-compass": "^1.0.4",
     "@mongodb-js/webpack-config-compass": "^1.3.9",
     "@segment/analytics-node": "^1.1.4",