From 1ec9c79af74b16f9e8995be18c6549d4f27caa92 Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Wed, 5 Apr 2023 17:10:01 +0100 Subject: [PATCH] tools: add root certificate update script Automates the steps from `doc/contributing/maintaining-root-certs.md`. Extend "Tools and deps update" workflow to use the new script to update the root certificates. --- .github/workflows/tools.yml | 11 +- doc/contributing/maintaining-root-certs.md | 15 ++ tools/dep_updaters/update-root-certs.mjs | 234 +++++++++++++++++++++ 3 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 tools/dep_updaters/update-root-certs.mjs diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index 0ee2cfc758f757..504f97836dad29 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -167,6 +167,11 @@ jobs: cat temp-output tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true rm temp-output + - id: root-certificates + subsystem: crypto + label: crypto, notable-change + run: | + node ./tools/dep_updaters/update-root-certs.mjs -v -f "$GITHUB_ENV" steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: @@ -174,6 +179,10 @@ jobs: - run: ${{ matrix.run }} env: GITHUB_TOKEN: ${{ secrets.GH_USER_TOKEN }} + - name: Generate commit message if not set + if: ${{ env.COMMIT_MSG == '' }} + run: | + echo "COMMIT_MSG=${{ matrix.subsystem }}: update ${{ matrix.id }} to ${{ env.NEW_VERSION }}" >> "$GITHUB_ENV" - uses: gr2m/create-or-update-pull-request-action@77596e3166f328b24613f7082ab30bf2d93079d5 # Creates a PR or update the Action's existing PR, or # no-op if the base branch is already up-to-date. @@ -183,6 +192,6 @@ jobs: author: Node.js GitHub Bot body: This is an automated update of ${{ matrix.id }} to ${{ env.NEW_VERSION }}. branch: actions/tools-update-${{ matrix.id }} # Custom branch *just* for this Action. - commit-message: '${{ matrix.subsystem }}: update ${{ matrix.id }} to ${{ env.NEW_VERSION }}' + commit-message: '${{ env.COMMIT_MSG }}' labels: ${{ matrix.label }} title: '${{ matrix.subsystem }}: update ${{ matrix.id }} to ${{ env.NEW_VERSION }}' diff --git a/doc/contributing/maintaining-root-certs.md b/doc/contributing/maintaining-root-certs.md index d7fb05a4f2485a..aff4b1cc20b7d8 100644 --- a/doc/contributing/maintaining-root-certs.md +++ b/doc/contributing/maintaining-root-certs.md @@ -15,6 +15,19 @@ check the [NSS release schedule][]. ## Process +The `tools/dep_updaters/update-root-certs.mjs` script automates the update of +the root certificates, including: + +* Downloading `certdata.txt` from Mozilla's source control repository. +* Running `tools/mk-ca-bundle.pl` to convert the certificates and generate + `src/node_root_certs.h`. +* Using `git diff-files` to determine which certificate have been added and/or + removed. + +Manual instructions are included in the following collapsed section. + +
+ Commands assume that the current working directory is the root of a checkout of the nodejs/node repository. @@ -121,5 +134,7 @@ the nodejs/node repository. - OpenTrust Root CA G3 ``` +
+ [NSS release schedule]: https://wiki.mozilla.org/NSS:Release_Versions [tag list]: https://hg.mozilla.org/projects/nss/tags diff --git a/tools/dep_updaters/update-root-certs.mjs b/tools/dep_updaters/update-root-certs.mjs new file mode 100644 index 00000000000000..99b74580369ed6 --- /dev/null +++ b/tools/dep_updaters/update-root-certs.mjs @@ -0,0 +1,234 @@ +// Script to update certdata.txt from NSS. +import { execFileSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { createWriteStream } from 'node:fs'; +import { basename, join, relative } from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +// Constants for NSS release metadata. +const kNSSVersion = 'version'; +const kNSSDate = 'date'; +const kFirefoxVersion = 'firefoxVersion'; +const kFirefoxDate = 'firefoxDate'; + +const __filename = fileURLToPath(import.meta.url); +const now = new Date(); + +const formatDate = (d) => { + const iso = d.toISOString(); + return iso.substring(0, iso.indexOf('T')); +}; + +const normalizeTD = (text) => { + // Remove whitespace and any HTML tags. + return text?.trim().replace(/<.*?>/g, ''); +}; +const getReleases = (text) => { + const releases = []; + const tableRE = /]+>([\S\s]*?)<\/table>/g; + const tableRowRE = /]*>([\S\s]*?)<\/tr>/g; + const tableHeaderRE = /
]*>([\S\s]*?)<\/th>/g; + const tableDataRE = /]*>([\S\s]*?)<\/td>/g; + for (const table of text.matchAll(tableRE)) { + const columns = {}; + const matches = table[1].matchAll(tableRowRE); + // First row has the table header. + let row = matches.next(); + if (row.done) { + continue; + } + const headers = Array.from(row.value[1].matchAll(tableHeaderRE), (m) => m[1]); + if (headers.length > 0) { + for (let i = 0; i < headers.length; i++) { + if (/NSS version/i.test(headers[i])) { + columns[kNSSVersion] = i; + } else if (/Release.*from branch/i.test(headers[i])) { + columns[kNSSDate] = i; + } else if (/Firefox version/i.test(headers[i])) { + columns[kFirefoxVersion] = i; + } else if (/Firefox release date/i.test(headers[i])) { + columns[kFirefoxDate] = i; + } + } + } + // Filter out "NSS Certificate bugs" table. + if (columns[kNSSDate] === undefined) { + continue; + } + // Scrape releases. + row = matches.next(); + while (!row.done) { + const cells = Array.from(row.value[1].matchAll(tableDataRE), (m) => m[1]); + const release = {}; + release[kNSSVersion] = normalizeTD(cells[columns[kNSSVersion]]); + release[kNSSDate] = new Date(normalizeTD(cells[columns[kNSSDate]])); + release[kFirefoxVersion] = normalizeTD(cells[columns[kFirefoxVersion]]); + release[kFirefoxDate] = new Date(normalizeTD(cells[columns[kFirefoxDate]])); + releases.push(release); + row = matches.next(); + } + } + return releases; +}; + +const getLatestVersion = (releases) => { + const arrayNumberSort = (x, y, i) => { + if (x[i] === undefined && y[i] === undefined) { + return 0; + } else if (x[i] === y[i]) { + return arrayNumberSort(x, y, i + 1); + } + return (x[i] ?? 0) - (y[i] ?? 0); + }; + const extractVersion = (t) => { + return t[kNSSVersion].split('.').map((n) => parseInt(n)); + }; + const releaseSorter = (x, y) => { + return arrayNumberSort(extractVersion(x), extractVersion(y), 0); + }; + return releases.sort(releaseSorter).filter(pastRelease).at(-1)[kNSSVersion]; +}; + +const pastRelease = (r) => { + return r[kNSSDate] < now; +}; + +const options = { + help: { + type: 'boolean', + }, + file: { + short: 'f', + type: 'string', + }, + verbose: { + short: 'v', + type: 'boolean', + }, +}; +const { + positionals, + values, +} = parseArgs({ + allowPositionals: true, + options, +}); + +if (values.help) { + console.log(`Usage: ${basename(__filename)} [OPTION]... [VERSION]...`); + console.log(); + console.log('Updates certdata.txt to NSS VERSION (most recent release by default).'); + console.log(''); + console.log(' -f, --file=FILE writes a commit message reflecting the change to the'); + console.log(' specified FILE'); + console.log(' -v, --verbose writes progress to stdout'); + console.log(' --help display this help and exit'); + process.exit(0); +} + +if (values.verbose) { + console.log('Fetching NSS release schedule'); +} +const scheduleURL = 'https://wiki.mozilla.org/NSS:Release_Versions'; +const schedule = await fetch(scheduleURL); +if (!schedule.ok) { + console.error(`Failed to fetch ${scheduleURL}: ${schedule.status}: ${schedule.statusText}`); + process.exit(-1); +} +const scheduleText = await schedule.text(); +const nssReleases = getReleases(scheduleText); + +// Retrieve metadata for the NSS release being updated to. +const version = positionals[0] ?? getLatestVersion(nssReleases); +const release = nssReleases.find((r) => { + return new RegExp(`^${version.replace('.', '\\.')}\\b`).test(r[kNSSVersion]); +}); +if (!pastRelease(release)) { + console.warn(`Warning: NSS ${version} is not due to be released until ${formatDate(release[kNSSDate])}`); +} +if (values.verbose) { + console.log('Found NSS version:'); + console.log(release); +} + +// Fetch certdata.txt and overwrite the local copy. +const tag = `NSS_${version.replaceAll('.', '_')}_RTM`; +const certdataURL = `https://hg.mozilla.org/projects/nss/raw-file/${tag}/lib/ckfw/builtins/certdata.txt`; +if (values.verbose) { + console.log(`Fetching ${certdataURL}`); +} +const checkoutDir = join(__filename, '..', '..', '..'); +const certdata = await fetch(certdataURL); +const certdataFile = join(checkoutDir, 'tools', 'certdata.txt'); +if (!certdata.ok) { + console.error(`Failed to fetch ${certdataURL}: ${certdata.status}: ${certdata.statusText}`); + process.exit(-1); +} +if (values.verbose) { + console.log(`Writing ${certdataFile}`); +} +await pipeline(certdata.body, createWriteStream(certdataFile)); + +// Run tools/mk-ca-bundle.pl to generate src/node_root_certs.h. +if (values.verbose) { + console.log('Running tools/mk-ca-bundle.pl'); +} +const opts = { encoding: 'utf8' }; +const mkCABundleTool = join(checkoutDir, 'tools', 'mk-ca-bundle.pl'); +const mkCABundleOut = execFileSync(mkCABundleTool, + values.verbose ? [ '-v' ] : [], + opts); +if (values.verbose) { + console.log(mkCABundleOut); +} + +// Determine certificates added and/or removed. +const certHeaderFile = relative(process.cwd(), join(checkoutDir, 'src', 'node_root_certs.h')); +const diff = execFileSync('git', [ 'diff-files', '-u', '--', certHeaderFile ], opts); +if (values.verbose) { + console.log(diff); +} +const certsAddedRE = /^\+\/\* (.*) \*\//gm; +const certsRemovedRE = /^-\/\* (.*) \*\//gm; +const added = [ ...diff.matchAll(certsAddedRE) ].map((m) => m[1]); +const removed = [ ...diff.matchAll(certsRemovedRE) ].map((m) => m[1]); + +const commitMsg = [ + `crypto: update root certificates to NSS ${release[kNSSVersion]}`, + '', + `This is the certdata.txt[0] from NSS ${release[kNSSVersion]}, released on ${formatDate(release[kNSSDate])}.`, + '', + `This is the version of NSS that ${release[kFirefoxDate] < now ? 'shipped' : 'will ship'} in Firefox ${release[kFirefoxVersion]} on`, + `${formatDate(release[kFirefoxDate])}.`, + '', +]; +if (added.length > 0) { + commitMsg.push('Certificates added:'); + commitMsg.push(...added.map((cert) => `- ${cert}`)); + commitMsg.push(''); +} +if (removed.length > 0) { + commitMsg.push('Certificates removed:'); + commitMsg.push(...removed.map((cert) => `- ${cert}`)); + commitMsg.push(''); +} +commitMsg.push(`[0] ${certdataURL}`); +const delimiter = randomUUID(); +const properties = [ + `NEW_VERSION=${release[kNSSVersion]}`, + `COMMIT_MSG<<${delimiter}`, + ...commitMsg, + delimiter, + '', +].join('\n'); +if (values.verbose) { + console.log(properties); +} +const propertyFile = values.file; +if (propertyFile !== undefined) { + console.log(`Writing to ${propertyFile}`); + await pipeline(Readable.from(properties), createWriteStream(propertyFile)); +}