[LOCAL] Download artifacts from CI to speed up testing (#37971) (#38612)

Co-authored-by: Riccardo Cipolleschi <cipolleschi@fb.com>
resolved: #37971

.circleci/config.yml

@@ -807,6 +807,10 @@ jobs:
       - report_bundle_size:
           platform: android
 
+      - store_artifacts:
+          path: ~/react-native/packages/rn-tester/android/app/build/outputs/apk/
+          destination: rntester-apk
+
       # Optionally, run disabled tests
       - when:
           condition: << parameters.run_disabled_tests >>
@@ -1053,7 +1057,10 @@ jobs:
 
       - run:
           name: Display Environment info
-          command: npx envinfo@latest
+          command: |
+            npm install -g envinfo
+            envinfo -v
+            envinfo
 
       - restore_cache:
           keys:

scripts/circle-ci-artifacts-utils.js

@@ -0,0 +1,198 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+'use strict';
+
+const {exec} = require('shelljs');
+
+const util = require('util');
+const asyncRequest = require('request');
+const request = util.promisify(asyncRequest);
+
+let circleCIHeaders;
+let jobs;
+let baseTemporaryPath;
+
+async function initialize(circleCIToken, baseTempPath, branchName) {
+  console.info('Getting CircleCI information');
+  circleCIHeaders = {'Circle-Token': circleCIToken};
+  baseTemporaryPath = baseTempPath;
+  exec(`mkdir -p ${baseTemporaryPath}`);
+  const pipeline = await _getLastCircleCIPipelineID(branchName);
+  const packageAndReleaseWorkflow = await _getPackageAndReleaseWorkflow(
+    pipeline.id,
+  );
+  const testsWorkflow = await _getTestsWorkflow(pipeline.id);
+  const jobsPromises = [
+    _getCircleCIJobs(packageAndReleaseWorkflow.id),
+    _getCircleCIJobs(testsWorkflow.id),
+  ];
+
+  const jobsResults = await Promise.all(jobsPromises);
+
+  jobs = jobsResults.flatMap(j => j);
+}
+
+function baseTmpPath() {
+  return baseTemporaryPath;
+}
+
+async function _getLastCircleCIPipelineID(branchName) {
+  const options = {
+    method: 'GET',
+    url: 'https://circleci.com/api/v2/project/gh/facebook/react-native/pipeline',
+    qs: {
+      branch: branchName,
+    },
+    headers: circleCIHeaders,
+  };
+
+  const response = await request(options);
+  if (response.error) {
+    throw new Error(error);
+  }
+
+  const items = JSON.parse(response.body).items;
+
+  if (!items || items.length === 0) {
+    throw new Error(
+      'No pipelines found on this branch. Make sure that the CI has run at least once, successfully',
+    );
+  }
+
+  const lastPipeline = items[0];
+  return {id: lastPipeline.id, number: lastPipeline.number};
+}
+
+async function _getSpecificWorkflow(pipelineId, workflowName) {
+  const options = {
+    method: 'GET',
+    url: `https://circleci.com/api/v2/pipeline/${pipelineId}/workflow`,
+    headers: circleCIHeaders,
+  };
+  const response = await request(options);
+  if (response.error) {
+    throw new Error(error);
+  }
+
+  const body = JSON.parse(response.body);
+  let workflow = body.items.find(w => w.name === workflowName);
+  _throwIfWorkflowNotFound(workflow, workflowName);
+  return workflow;
+}
+
+function _throwIfWorkflowNotFound(workflow, name) {
+  if (!workflow) {
+    throw new Error(
+      `Can't find a workflow named ${name}. Please check whether that workflow has started.`,
+    );
+  }
+}
+
+async function _getPackageAndReleaseWorkflow(pipelineId) {
+  return _getSpecificWorkflow(pipelineId, 'package_and_publish_release_dryrun');
+}
+
+async function _getTestsWorkflow(pipelineId) {
+  return _getSpecificWorkflow(pipelineId, 'tests');
+}
+
+async function _getCircleCIJobs(workflowId) {
+  const options = {
+    method: 'GET',
+    url: `https://circleci.com/api/v2/workflow/${workflowId}/job`,
+    headers: circleCIHeaders,
+  };
+  const response = await request(options);
+  if (response.error) {
+    throw new Error(error);
+  }
+
+  const body = JSON.parse(response.body);
+  return body.items;
+}
+
+async function _getJobsArtifacts(jobNumber) {
+  const options = {
+    method: 'GET',
+    url: `https://circleci.com/api/v2/project/gh/facebook/react-native/${jobNumber}/artifacts`,
+    headers: circleCIHeaders,
+  };
+  const response = await request(options);
+  if (response.error) {
+    throw new Error(error);
+  }
+
+  const body = JSON.parse(response.body);
+  return body.items;
+}
+
+async function _findUrlForJob(jobName, artifactPath) {
+  const job = jobs.find(j => j.name === jobName);
+  _throwIfJobIsNull(job);
+  _throwIfJobIsUnsuccessful(job);
+
+  const artifacts = await _getJobsArtifacts(job.job_number);
+  return artifacts.find(artifact => artifact.path.indexOf(artifactPath) > -1)
+    .url;
+}
+
+function _throwIfJobIsNull(job) {
+  if (!job) {
+    throw new Error(
+      `Can't find a job with name ${job.name}. Please verify that it has been executed and that all its dependencies completed successfully.`,
+    );
+  }
+}
+
+function _throwIfJobIsUnsuccessful(job) {
+  if (job.status !== 'success') {
+    throw new Error(
+      `The job ${job.name} status is ${job.status}. We need a 'success' status to proceed with the testing.`,
+    );
+  }
+}
+
+async function artifactURLHermesDebug() {
+  return _findUrlForJob('build_hermes_macos-Debug', 'hermes-ios-debug.tar.gz');
+}
+
+async function artifactURLForMavenLocal() {
+  return _findUrlForJob('build_and_publish_npm_package-2', 'maven-local.zip');
+}
+
+async function artifactURLForHermesRNTesterAPK(emulatorArch) {
+  return _findUrlForJob(
+    'test_android',
+    `rntester-apk/hermes/debug/app-hermes-${emulatorArch}-debug.apk`,
+  );
+}
+
+async function artifactURLForJSCRNTesterAPK(emulatorArch) {
+  return _findUrlForJob(
+    'test_android',
+    `rntester-apk/jsc/debug/app-jsc-${emulatorArch}-debug.apk`,
+  );
+}
+
+function downloadArtifact(artifactURL, destination) {
+  exec(`rm -rf ${destination}`);
+  exec(`curl ${artifactURL} -Lo ${destination}`);
+}
+
+module.exports = {
+  initialize,
+  downloadArtifact,
+  artifactURLForJSCRNTesterAPK,
+  artifactURLForHermesRNTesterAPK,
+  artifactURLForMavenLocal,
+  artifactURLHermesDebug,
+  baseTmpPath,
+};

scripts/npm-utils.js

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+'use strict';
+
+/**
+ * `package` is an object form of package.json
+ * `dependencies` is a map of dependency to version string
+ *
+ * This replaces both dependencies and devDependencies in package.json
+ */
+function applyPackageVersions(originalPackageJson, packageVersions) {
+  const packageJson = {...originalPackageJson};
+
+  for (const name of Object.keys(packageVersions)) {
+    if (
+      packageJson.dependencies != null &&
+      packageJson.dependencies[name] != null
+    ) {
+      packageJson.dependencies[name] = packageVersions[name];
+    }
+
+    if (
+      packageJson.devDependencies != null &&
+      packageJson.devDependencies[name] != null
+    ) {
+      packageJson.devDependencies[name] = packageVersions[name];
+    }
+  }
+  return packageJson;
+}
+
+module.exports = {
+  applyPackageVersions,
+};

scripts/release-utils.js

@@ -111,11 +111,6 @@ function generateiOSArtifacts(
 ) {
   pushd(`${hermesCoreSourceFolder}`);
 
-  //Need to generate hermesc
-  exec(
-    `${hermesCoreSourceFolder}/utils/build-hermesc-xcode.sh ${hermesCoreSourceFolder}/build_host_hermesc`,
-  );
-
   //Generating iOS Artifacts
   exec(
     `JSI_PATH=${jsiFolder} BUILD_TYPE=${buildType} ${hermesCoreSourceFolder}/utils/build-mac-framework.sh`,

scripts/test-e2e-local-clean.js

@@ -44,6 +44,7 @@ if (isPackagerRunning() === 'running') {
 console.info('\n** Cleaning Gradle build artifacts **\n');
 exec('./gradlew cleanAll');
 exec('rm -rf /tmp/maven-local');
+exec('rm -rf /tmp/react-native-tmp');
 
 // iOS
 console.info('\n** Nuking the derived data folder **\n');
@@ -56,9 +57,6 @@ exec('rm -rf ~/Library/Caches/CocoaPods/Pods/External/hermes-engine');
 console.info('\n** Removing the RNTester Pods **\n');
 exec('rm -rf packages/rn-tester/Pods');
 
-// I'm not sure we want to also remove the lock file
-// exec('rm -rf packages/rn-tester/Podfile.lock');
-
 // RNTestProject
 console.info('\n** Removing the RNTestProject folder **\n');
 exec('rm -rf /tmp/RNTestProject');