Enhance darwin-arm64 release (opensearch-project#5455)

* Add min-builds for `darwin-x64` and `darwin-arm64`. * Expose `--darwin-arm` as a parameter of the build script. * Work around build's cleanup failure when removal of an already deleted folder is attempted. * Exclude bundling fallbacks with `darwin-arm64` since Node.js v14 has no releases for them. * Update tests to accommodate `darwin-arm64`. * Generalize the Node.js versions used in some tests. Signed-off-by: Miki <miki@amazon.com>
AMoo-Miki · Nov 10, 2023 · 89dc051 · 89dc051
1 parent b82aa1c
commit 89dc051
Show file tree

Hide file tree

Showing 13 changed files with 140 additions and 92 deletions.
diff --git a/.github/workflows/build_and_test_workflow.yml b/.github/workflows/build_and_test_workflow.yml
@@ -281,7 +281,7 @@ jobs:
       - name: Build plugins
         run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 10 --scan-dir "./test/plugin_functional/plugins"
 
-      - name: Run functional plugin tests 
+      - name: Run functional plugin tests
         id: plugin-ftr-tests
         run: node scripts/functional_tests.js --config test/plugin_functional/config.ts
 
@@ -308,6 +308,16 @@ jobs:
             ext: tar.gz
             suffix: linux-arm64
             script: build-platform --linux-arm --skip-os-packages
+          - os: macos-latest
+            name: macOS x64
+            ext: tar.gz
+            suffix: darwin-x64
+            script: build-platform --darwin --skip-os-packages
+          - os: macos-latest
+            name: macOS ARM64
+            ext: tar.gz
+            suffix: darwin-arm64
+            script: build-platform --darwin-arm --skip-os-packages
           - os: windows-latest
             name: Windows x64
             ext: zip

diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts
@@ -60,6 +60,7 @@ it('build dist for current platform, without packages, by default', () => {
         "targetAllPlatforms": false,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -89,6 +90,7 @@ it('build dist for linux x64 platform, without packages, if --linux is passed',
         "targetAllPlatforms": false,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": true,
           "linuxArm": false,
           "windows": false,
@@ -118,6 +120,7 @@ it('build dist for linux arm64 platform, without packages, if --linux-arm is pas
         "targetAllPlatforms": false,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": true,
           "windows": false,
@@ -147,6 +150,7 @@ it('build dist for darwin x64 platform, without packages, if --darwin is passed'
         "targetAllPlatforms": false,
         "targetPlatforms": Object {
           "darwin": true,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -176,6 +180,7 @@ it('build dist for windows x64 platform, without packages, if --windows is passe
         "targetAllPlatforms": false,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": true,
@@ -205,6 +210,7 @@ it('builds packages if --all-platforms is passed', () => {
         "targetAllPlatforms": true,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -234,6 +240,7 @@ it('limits packages if --rpm passed with --all-platforms', () => {
         "targetAllPlatforms": true,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -263,6 +270,7 @@ it('limits packages if --deb passed with --all-platforms', () => {
         "targetAllPlatforms": true,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -293,6 +301,7 @@ it('limits packages if --docker passed with --all-platforms', () => {
         "targetAllPlatforms": true,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,
@@ -323,6 +332,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
         "targetAllPlatforms": true,
         "targetPlatforms": Object {
           "darwin": false,
+          "darwinArm": false,
           "linux": false,
           "linuxArm": false,
           "windows": false,

diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts
@@ -52,6 +52,7 @@ export function readCliArgs(argv: string[]) {
       'all-platforms',
       'windows',
       'darwin',
+      'darwin-arm',
       'linux',
       'linux-arm',
       'verbose',
@@ -133,6 +134,7 @@ export function readCliArgs(argv: string[]) {
     targetPlatforms: {
       windows: Boolean(flags.windows),
       darwin: Boolean(flags.darwin),
+      darwinArm: Boolean(flags['darwin-arm']),
       linux: Boolean(flags.linux),
       linuxArm: Boolean(flags['linux-arm']),
     },

diff --git a/src/dev/build/cli.ts b/src/dev/build/cli.ts
@@ -61,6 +61,7 @@ if (showHelp) {
         --linux                 {dim Produce archives only for linux x64 platform}
         --linux-arm             {dim Produce archives only for linux arm64 platform}
         --darwin                {dim Produce archives only for darwin x64 platform}
+        --darwin-arm            {dim Produce archives only for darwin arm64 platform}
         --windows               {dim Produce archives only for windows x64 platform}
         --rpm                   {dim Only build the rpm package}
         --deb                   {dim Only build the deb package}

diff --git a/src/dev/build/lib/fs.ts b/src/dev/build/lib/fs.ts
@@ -29,6 +29,7 @@
  */
 
 import fs from 'fs';
+import { rm } from 'fs/promises';
 import { createHash } from 'crypto';
 import { pipeline, Writable } from 'stream';
 import { resolve, dirname, isAbsolute, sep } from 'path';
@@ -138,8 +139,7 @@ export async function deleteEmptyFolders(
   );
   assertAbsolute(rootFolderPath.startsWith('!') ? rootFolderPath.slice(1) : rootFolderPath);
 
-  // Delete empty is used to gather all the empty folders and
-  // then we use del to actually delete them
+  // `deleteEmpty` is used to gather all the empty folders then `rm` is used to actually delete them
   const emptyFoldersList = await deleteEmpty(rootFolderPath, {
     // @ts-expect-error DT package has incorrect types https://github.com/jonschlinkert/delete-empty/blob/6ae34547663e6845c3c98b184c606fa90ef79c0a/index.js#L160
     dryRun: true,
@@ -148,12 +148,11 @@ export async function deleteEmptyFolders(
   const foldersToDelete = emptyFoldersList.filter((folderToDelete) => {
     return !foldersToKeep.some((folderToKeep) => folderToDelete.includes(folderToKeep));
   });
-  const deletedEmptyFolders = await del(foldersToDelete, {
-    concurrency: 4,
-  });
 
-  log.debug('Deleted %d empty folders', deletedEmptyFolders.length);
-  log.verbose('Deleted:', longInspect(deletedEmptyFolders));
+  await Promise.all(foldersToDelete.map((folder) => rm(folder, { force: true, recursive: true })));
+
+  log.debug('Deleted %d empty folders', foldersToDelete.length);
+  log.verbose('Deleted:', longInspect(foldersToDelete));
 }
 
 interface CopyOptions {

diff --git a/src/dev/build/tasks/create_archives_sources_task.ts b/src/dev/build/tasks/create_archives_sources_task.ts
@@ -56,17 +56,21 @@ export const CreateArchivesSources: Task = {
 
         // ToDo [NODE14]: Remove this Node.js 14 fallback download
         // Copy the Node.js 14 binaries into node/fallback to be used by `use_node`
-        await scanCopy({
-          source: (
-            await getNodeVersionDownloadInfo(
-              NODE14_FALLBACK_VERSION,
-              platform.getNodeArch(),
-              platform.isWindows(),
-              config.resolveFromRepo()
-            )
-          ).extractDir,
-          destination: build.resolvePathForPlatform(platform, 'node', 'fallback'),
-        });
+        if (platform.getBuildName() === 'darwin-arm64') {
+          log.warning(`There are no fallback Node.js versions released for darwin-arm64.`);
+        } else {
+          await scanCopy({
+            source: (
+              await getNodeVersionDownloadInfo(
+                NODE14_FALLBACK_VERSION,
+                platform.getNodeArch(),
+                platform.isWindows(),
+                config.resolveFromRepo()
+              )
+            ).extractDir,
+            destination: build.resolvePathForPlatform(platform, 'node', 'fallback'),
+          });
+        }
 
         log.debug('Node.js copied into', platform.getNodeArch(), 'specific build directory');
       })

diff --git a/src/dev/build/tasks/nodejs/__snapshots__/download_node_builds_task.test.ts.snap b/src/dev/build/tasks/nodejs/__snapshots__/download_node_builds_task.test.ts.snap
diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts
@@ -36,13 +36,15 @@ import {
 
 import { Config, Platform } from '../../lib';
 import { DownloadNodeBuilds } from './download_node_builds_task';
+import { stripAnsiSnapshotSerializer } from '../../../../core/test_helpers/strip_ansi_snapshot_serializer';
 
 jest.mock('./node_shasums');
 jest.mock('./node_download_info');
 jest.mock('../../lib/download');
 jest.mock('../../lib/get_build_number');
 
 expect.addSnapshotSerializer(createAnyInstanceSerializer(ToolingLog));
+expect.addSnapshotSerializer(stripAnsiSnapshotSerializer);
 
 const { getNodeDownloadInfo, getNodeVersionDownloadInfo } = jest.requireMock(
   './node_download_info'
@@ -67,6 +69,8 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) {
       linux: false,
       linuxArm: false,
       darwin: false,
+      darwinArm: false,
+      windows: false,
     },
   });
 
@@ -78,19 +82,23 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) {
     };
   });
 
-  getNodeVersionDownloadInfo.mockImplementation((version, architecture, isWindows, repoRoot) => {
-    return {
-      url: `https://mirrors.nodejs.org/dist/v${version}/node-v${version}-${architecture}.tar.gz`,
-      downloadName: `node-v${version}-${architecture}.tar.gz`,
-      downloadPath: `/mocked/path/.node_binaries/${version}/node-v${version}-${architecture}.tar.gz`,
-      extractDir: `/mocked/path/.node_binaries/${version}/${architecture}`,
-      version,
-    };
-  });
+  getNodeVersionDownloadInfo.mockImplementation(
+    (version: string, architecture: string, isWindows: boolean, repoRoot: string) => {
+      return {
+        url: `https://mirrors.nodejs.org/dist/v${version}/node-v${version}-${architecture}.tar.gz`,
+        downloadName: `node-v${version}-${architecture}.tar.gz`,
+        downloadPath: `/mocked/path/.node_binaries/${version}/node-v${version}-${architecture}.tar.gz`,
+        extractDir: `/mocked/path/.node_binaries/${version}/${architecture}`,
+        version,
+      };
+    }
+  );
 
   getNodeShasums.mockReturnValue({
     'linux:downloadName': 'linux:sha256',
+    'linux-arm64:downloadName': 'linux-arm64:sha256',
     'darwin:downloadName': 'darwin:sha256',
+    'darwin-arm64:downloadName': 'darwin-arm64:sha256',
     'win32:downloadName': 'win32:sha256',
   });
 
@@ -182,15 +190,6 @@ it('downloads node builds for each platform', async () => {
           "url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-darwin-x64.tar.gz",
         },
       ],
-      Array [
-        Object {
-          "destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-darwin-arm64.tar.gz",
-          "log": <ToolingLog>,
-          "retries": 3,
-          "sha256": undefined,
-          "url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-darwin-arm64.tar.gz",
-        },
-      ],
       Array [
         Object {
           "destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-win32-x64.tar.gz",
@@ -202,7 +201,10 @@ it('downloads node builds for each platform', async () => {
       ],
     ]
   `);
-  expect(testWriter.messages).toMatchInlineSnapshot(`Array []`);
+  /* ToDo [NODE14]: Replace when Node.js 14 support is removed
+   * expect(testWriter.messages).toMatchInlineSnapshot(`Array []`);
+   */
+  expect(testWriter.messages).toMatchSnapshot();
 });
 
 it('rejects if any download fails', async () => {
@@ -211,5 +213,8 @@ it('rejects if any download fails', async () => {
   await expect(DownloadNodeBuilds.run(config, log, [])).rejects.toMatchInlineSnapshot(
     `[Error: Download failed for reasons]`
   );
-  expect(testWriter.messages).toMatchInlineSnapshot(`Array []`);
+  /* ToDo [NODE14]: Replace when Node.js 14 support is removed
+   * expect(testWriter.messages).toMatchInlineSnapshot(`Array []`);
+   */
+  expect(testWriter.messages).toMatchSnapshot();
 });
diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.ts b/src/dev/build/tasks/nodejs/download_node_builds_task.ts
@@ -60,6 +60,10 @@ export const DownloadNodeBuilds: GlobalTask = {
       }),
       // ToDo [NODE14]: Remove this Node.js 14 fallback download
       ...config.getTargetPlatforms().map(async (platform) => {
+        if (platform.getBuildName() === 'darwin-arm64') {
+          log.warning(`There are no fallback Node.js versions released for darwin-arm64.`);
+          return;
+        }
         const { url, downloadPath, downloadName } = await getNodeVersionDownloadInfo(
           NODE14_FALLBACK_VERSION,
           platform.getNodeArch(),

diff --git a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts
@@ -58,6 +58,7 @@ async function setup() {
       linux: false,
       linuxArm: false,
       darwin: false,
+      darwinArm: false,
       windows: false,
     },
   });
@@ -151,13 +152,6 @@ it('runs expected fs operations', async () => {
             "strip": 1,
           },
         ],
-        Array [
-          <absolute path>/.node_binaries/14.21.3/node-v14.21.3-darwin-arm64.tar.gz,
-          <absolute path>/.node_binaries/14.21.3/darwin-arm64,
-          Object {
-            "strip": 1,
-          },
-        ],
       ],
       "unzip": Array [
         Array [

diff --git a/src/dev/build/tasks/nodejs/extract_node_builds_task.ts b/src/dev/build/tasks/nodejs/extract_node_builds_task.ts
@@ -38,7 +38,7 @@ import {
 export const ExtractNodeBuilds: GlobalTask = {
   global: true,
   description: 'Extracting node.js builds for all platforms',
-  async run(config) {
+  async run(config, log) {
     await Promise.all([
       ...config.getTargetPlatforms().map(async (platform) => {
         const { downloadPath, extractDir } = await getNodeDownloadInfo(config, platform);
@@ -50,6 +50,10 @@ export const ExtractNodeBuilds: GlobalTask = {
       }),
       // ToDo [NODE14]: Remove this Node.js 14 fallback download
       ...config.getTargetPlatforms().map(async (platform) => {
+        if (platform.getBuildName() === 'darwin-arm64') {
+          log.warning(`There are no fallback Node.js versions released for darwin-arm64.`);
+          return;
+        }
         const { downloadPath, extractDir } = await getNodeVersionDownloadInfo(
           NODE14_FALLBACK_VERSION,
           platform.getNodeArch(),