From 497878642c43b5b07812ee24cf0079145139e73c Mon Sep 17 00:00:00 2001 From: Josh Romero Date: Tue, 30 May 2023 12:00:39 -0700 Subject: [PATCH 1/2] chore (lychee): Add company.net to exclusion list (#4171) Signed-off-by: Josh Romero --- .lycheeexclude | 1 + 1 file changed, 1 insertion(+) diff --git a/.lycheeexclude b/.lycheeexclude index 252db4e82787..35ae861e8f91 100644 --- a/.lycheeexclude +++ b/.lycheeexclude @@ -89,6 +89,7 @@ http://www.opensearch.org/painlessDocs https://www.hostedgraphite.com/ https://connectionurl.com http://169.254.169.254/latest/meta-data/ +http://company.net/* # External urls https://www.zeek.org/ From 20c928fccf9b2516b36613874b335e9291b69f81 Mon Sep 17 00:00:00 2001 From: Miki Date: Tue, 30 May 2023 13:50:54 -0700 Subject: [PATCH 2/2] Bundle Node 14 as a fallback for operating systems that cannot run Node 18 (#4151) Signed-off-by: ananzh Signed-off-by: Miki --- CHANGELOG.md | 1 + scripts/use_node | 10 +++- .../tasks/create_archives_sources_task.ts | 16 +++++- .../nodejs/download_node_builds_task.test.ts | 50 ++++++++++++++++++- .../tasks/nodejs/download_node_builds_task.ts | 35 +++++++++++-- .../nodejs/extract_node_builds_task.test.ts | 28 +++++++++++ .../tasks/nodejs/extract_node_builds_task.ts | 28 +++++++++-- .../build/tasks/nodejs/node_download_info.ts | 27 +++++++++- 8 files changed, 180 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e432e83d934..3a7acdd0a965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - [Table Visualization] Move format table, consolidate types and add unit tests ([#3397](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3397)) - [Multiple Datasource] Support Amazon OpenSearch Serverless ([#3957](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3957)) - Add support for Node.js >=14.20.1 <19 ([#4071](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4071)) +- Bundle Node.js 14 as a fallback for operating systems that cannot run Node.js 18 ([#4151](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4151)) ### 🐛 Bug Fixes diff --git a/scripts/use_node b/scripts/use_node index f32ecbcb833a..48e7e2858200 100755 --- a/scripts/use_node +++ b/scripts/use_node @@ -57,11 +57,17 @@ elif [ ! -z "$NODE_HOME" ]; then NODE_ERROR_MSG="in NODE_HOME" NODE_ERROR_SHOW=true else + # Set these variables outside, as catchalls, to show meaningful errors if needed NODE="$OSD_HOME/node/bin/node" NODE_ERROR_MSG="bundled with OpenSearch Dashboards" # A bin folder at the root is only present in release builds that have a bundled Node.js binary - if [ -x "OSD_HOME/bin" ]; then + if [ -d "${OSD_HOME}/bin" ]; then NODE_ERROR_SHOW=true + # Not all operating systems can run the latest Node.js and the fallback is for them + "${NODE}" -v 2> /dev/null + if [ $? -ne 0 ] && [ -d "${OSD_HOME}/node/fallback" ]; then + NODE="$OSD_HOME/node/fallback/bin/node" + fi fi fi @@ -75,7 +81,7 @@ else fi if [ ! -x "$NODE" ]; then - # Irrespective of NODE_ERROR_SHOW, show the error + # Irrespective of NODE_ERROR_SHOW, if NODE is not found or executable, show the error echo "Could not find a Node.js runtime binary $NODE_ERROR_MSG or on the system" >&2 exit 1 fi diff --git a/src/dev/build/tasks/create_archives_sources_task.ts b/src/dev/build/tasks/create_archives_sources_task.ts index bf53c2915560..55d9b5313f12 100644 --- a/src/dev/build/tasks/create_archives_sources_task.ts +++ b/src/dev/build/tasks/create_archives_sources_task.ts @@ -29,7 +29,7 @@ */ import { scanCopy, Task } from '../lib'; -import { getNodeDownloadInfo } from './nodejs'; +import { getNodeDownloadInfo, getNodeVersionDownloadInfo, NODE14_FALLBACK_VERSION } from './nodejs'; export const CreateArchivesSources: Task = { description: 'Creating platform-specific archive source directories', @@ -54,6 +54,20 @@ export const CreateArchivesSources: Task = { destination: build.resolvePathForPlatform(platform, 'node'), }); + // ToDo [NODE14]: Remove this Node.js 14 fallback download + // Copy the Node.js 14 binaries into node/fallback to be used by `use_node` + await scanCopy({ + source: ( + await getNodeVersionDownloadInfo( + NODE14_FALLBACK_VERSION, + platform.getNodeArch(), + platform.isWindows(), + config.resolveFromRepo() + ) + ).extractDir, + destination: build.resolvePathForPlatform(platform, 'node', 'fallback'), + }); + log.debug('Node.js copied into', platform.getNodeArch(), 'specific build directory'); }) ); diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts index 3aff61e7cf4f..f5905534e121 100644 --- a/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts @@ -44,7 +44,9 @@ jest.mock('../../lib/get_build_number'); expect.addSnapshotSerializer(createAnyInstanceSerializer(ToolingLog)); -const { getNodeDownloadInfo } = jest.requireMock('./node_download_info'); +const { getNodeDownloadInfo, getNodeVersionDownloadInfo } = jest.requireMock( + './node_download_info' +); const { getNodeShasums } = jest.requireMock('./node_shasums'); const { download } = jest.requireMock('../../lib/download'); @@ -76,6 +78,16 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) { }; }); + getNodeVersionDownloadInfo.mockImplementation((version, architecture, isWindows, repoRoot) => { + return { + url: `https://mirrors.nodejs.org/dist/v${version}/node-v${version}-${architecture}.tar.gz`, + downloadName: `node-v${version}-${architecture}.tar.gz`, + downloadPath: `/mocked/path/.node_binaries/${version}/node-v${version}-${architecture}.tar.gz`, + extractDir: `/mocked/path/.node_binaries/${version}/${architecture}`, + version, + }; + }); + getNodeShasums.mockReturnValue({ 'linux:downloadName': 'linux:sha256', 'darwin:downloadName': 'darwin:sha256', @@ -134,6 +146,42 @@ it('downloads node builds for each platform', async () => { "url": "win32:url", }, ], + Array [ + Object { + "destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-linux-x64.tar.gz", + "log": , + "retries": 3, + "sha256": undefined, + "url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-linux-x64.tar.gz", + }, + ], + Array [ + Object { + "destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-linux-arm64.tar.gz", + "log": , + "retries": 3, + "sha256": undefined, + "url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-linux-arm64.tar.gz", + }, + ], + Array [ + Object { + "destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-darwin-x64.tar.gz", + "log": , + "retries": 3, + "sha256": undefined, + "url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-darwin-x64.tar.gz", + }, + ], + Array [ + Object { + "destination": "/mocked/path/.node_binaries/14.21.3/node-v14.21.3-win32-x64.tar.gz", + "log": , + "retries": 3, + "sha256": undefined, + "url": "https://mirrors.nodejs.org/dist/v14.21.3/node-v14.21.3-win32-x64.tar.gz", + }, + ], ] `); expect(testWriter.messages).toMatchInlineSnapshot(`Array []`); diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.ts b/src/dev/build/tasks/nodejs/download_node_builds_task.ts index 2b3ca54dce29..393a02176e17 100644 --- a/src/dev/build/tasks/nodejs/download_node_builds_task.ts +++ b/src/dev/build/tasks/nodejs/download_node_builds_task.ts @@ -30,7 +30,12 @@ import { download, GlobalTask } from '../../lib'; import { getNodeShasums } from './node_shasums'; -import { getNodeDownloadInfo, getRequiredVersion } from './node_download_info'; +import { + getNodeDownloadInfo, + getNodeVersionDownloadInfo, + getRequiredVersion, + NODE14_FALLBACK_VERSION, +} from './node_download_info'; export const DownloadNodeBuilds: GlobalTask = { global: true, @@ -38,8 +43,12 @@ export const DownloadNodeBuilds: GlobalTask = { async run(config, log) { const requiredNodeVersion = getRequiredVersion(config); const shasums = await getNodeShasums(log, requiredNodeVersion); - await Promise.all( - config.getTargetPlatforms().map(async (platform) => { + + // ToDo [NODE14]: Remove this Node.js 14 fallback download + const node14ShaSums = await getNodeShasums(log, NODE14_FALLBACK_VERSION); + + await Promise.all([ + ...config.getTargetPlatforms().map(async (platform) => { const { url, downloadPath, downloadName } = await getNodeDownloadInfo(config, platform); await download({ log, @@ -48,7 +57,23 @@ export const DownloadNodeBuilds: GlobalTask = { destination: downloadPath, retries: 3, }); - }) - ); + }), + // ToDo [NODE14]: Remove this Node.js 14 fallback download + ...config.getTargetPlatforms().map(async (platform) => { + const { url, downloadPath, downloadName } = await getNodeVersionDownloadInfo( + NODE14_FALLBACK_VERSION, + platform.getNodeArch(), + platform.isWindows(), + config.resolveFromRepo() + ); + await download({ + log, + url, + sha256: node14ShaSums[downloadName], + destination: downloadPath, + retries: 3, + }); + }), + ]); }, }; diff --git a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts index 10261c447bef..e68539310903 100644 --- a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts @@ -123,6 +123,27 @@ it('runs expected fs operations', async () => { "strip": 1, }, ], + Array [ + /.node_binaries/14.21.3/node-v14.21.3-linux-x64.tar.gz, + /.node_binaries/14.21.3/linux-x64, + Object { + "strip": 1, + }, + ], + Array [ + /.node_binaries/14.21.3/node-v14.21.3-linux-arm64.tar.gz, + /.node_binaries/14.21.3/linux-arm64, + Object { + "strip": 1, + }, + ], + Array [ + /.node_binaries/14.21.3/node-v14.21.3-darwin-x64.tar.gz, + /.node_binaries/14.21.3/darwin-x64, + Object { + "strip": 1, + }, + ], ], "unzip": Array [ Array [ @@ -132,6 +153,13 @@ it('runs expected fs operations', async () => { "strip": 1, }, ], + Array [ + /.node_binaries/14.21.3/node-v14.21.3-win-x64.zip, + /.node_binaries/14.21.3/win32-x64, + Object { + "strip": 1, + }, + ], ], } `); diff --git a/src/dev/build/tasks/nodejs/extract_node_builds_task.ts b/src/dev/build/tasks/nodejs/extract_node_builds_task.ts index 28b2ebe24d4b..7934718c0bce 100644 --- a/src/dev/build/tasks/nodejs/extract_node_builds_task.ts +++ b/src/dev/build/tasks/nodejs/extract_node_builds_task.ts @@ -29,21 +29,39 @@ */ import { untar, unzip, GlobalTask } from '../../lib'; -import { getNodeDownloadInfo } from './node_download_info'; +import { + getNodeDownloadInfo, + getNodeVersionDownloadInfo, + NODE14_FALLBACK_VERSION, +} from './node_download_info'; export const ExtractNodeBuilds: GlobalTask = { global: true, description: 'Extracting node.js builds for all platforms', async run(config) { - await Promise.all( - config.getTargetPlatforms().map(async (platform) => { + await Promise.all([ + ...config.getTargetPlatforms().map(async (platform) => { const { downloadPath, extractDir } = await getNodeDownloadInfo(config, platform); if (platform.isWindows()) { await unzip(downloadPath, extractDir, { strip: 1 }); } else { await untar(downloadPath, extractDir, { strip: 1 }); } - }) - ); + }), + // ToDo [NODE14]: Remove this Node.js 14 fallback download + ...config.getTargetPlatforms().map(async (platform) => { + const { downloadPath, extractDir } = await getNodeVersionDownloadInfo( + NODE14_FALLBACK_VERSION, + platform.getNodeArch(), + platform.isWindows(), + config.resolveFromRepo() + ); + if (platform.isWindows()) { + await unzip(downloadPath, extractDir, { strip: 1 }); + } else { + await untar(downloadPath, extractDir, { strip: 1 }); + } + }), + ]); }, }; diff --git a/src/dev/build/tasks/nodejs/node_download_info.ts b/src/dev/build/tasks/nodejs/node_download_info.ts index 26c56fc8e907..37eba4c29a58 100644 --- a/src/dev/build/tasks/nodejs/node_download_info.ts +++ b/src/dev/build/tasks/nodejs/node_download_info.ts @@ -28,7 +28,7 @@ * under the License. */ -import { basename } from 'path'; +import { basename, resolve } from 'path'; import fetch from 'node-fetch'; import semver from 'semver'; @@ -36,6 +36,8 @@ import { Config, Platform } from '../../lib'; const NODE_RANGE_CACHE: { [key: string]: string } = {}; +export const NODE14_FALLBACK_VERSION = '14.21.3'; + export async function getNodeDownloadInfo(config: Config, platform: Platform) { const version = getRequiredVersion(config); const arch = platform.getNodeArch(); @@ -57,6 +59,29 @@ export async function getNodeDownloadInfo(config: Config, platform: Platform) { }; } +export async function getNodeVersionDownloadInfo( + version: string, + architecture: string, + isWindows: boolean, + repoRoot: string +) { + const downloadName = isWindows + ? `node-v${version}-win-x64.zip` + : `node-v${version}-${architecture}.tar.gz`; + + const url = `https://mirrors.nodejs.org/dist/v${version}/${downloadName}`; + const downloadPath = resolve(repoRoot, '.node_binaries', version, basename(downloadName)); + const extractDir = resolve(repoRoot, '.node_binaries', version, architecture); + + return { + url, + downloadName, + downloadPath, + extractDir, + version, + }; +} + export async function getLatestNodeVersion(config: Config) { const range = config.getNodeRange(); // Check cache and return if known