Skip to content

Commit

Permalink
Run through all actions for each distFlavor in amp release before t…
Browse files Browse the repository at this point in the history 
…he next (#34569)

Co-authored-by: Raghu Simha <rsimha@amp.dev>
  • Loading branch information
@danielrozenberg @rsimha
danielrozenberg and rsimha authored May 27, 2021
1 parent 3b2531b commit c5a5cf5
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 109 deletions.
2 changes: 1 addition & 1 deletion .eslintignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ dist.tools
export
examples/storybook
extensions/**/dist
release
/release
result-reports
src/purifier/dist
test/coverage
Expand Down
2 changes: 1 addition & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ dist.tools
export
examples/storybook
extensions/**/dist
release
/release
result-reports
src/purifier/dist
test/coverage
Expand Down
2 changes: 1 addition & 1 deletion .prettierignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ dist.tools
export
examples/storybook
extensions/**/dist
release
/release
result-reports
src/purifier/dist
test/coverage
Expand Down
9 changes: 8 additions & 1 deletion build-system/tasks/clean.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ const ROOT_DIR = path.resolve(__dirname, '../../');

/**
* Cleans up various cache and output directories. Optionally cleans up inner
* node_modules package directories.
* node_modules package directories, or excludes some directories from deletion.
*/
async function clean() {
const pathsToDelete = [
Expand Down Expand Up @@ -61,6 +61,12 @@ async function clean() {
if (argv.include_subpackages) {
pathsToDelete.push('**/node_modules', '!node_modules');
}
if (argv.exclude) {
const excludes = argv.exclude.split(',');
for (const exclude of excludes) {
pathsToDelete.push(`!${exclude}`);
}
}
const deletedPaths = await del(pathsToDelete, {
expandDirectories: false,
dryRun: argv.dry_run,
Expand All @@ -82,4 +88,5 @@ clean.flags = {
'dry_run': 'Does a dry run without actually deleting anything',
'include_subpackages':
'Also cleans up inner node_modules package directories',
'exclude': 'Comma separated list of directories to exclude from deletion',
};
218 changes: 113 additions & 105 deletions build-system/tasks/release/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,37 @@
*/
'use strict';

/**
* @typedef {{
* name?: string,
* environment?: string,
* issue?: string,
* expiration_date_utc?: string,
* define_experiment_constant?: string,
* }}
*/
let ExperimentConfigDef;

/**
* @typedef {{
* experimentA: ExperimentConfigDef,
* experimentB: ExperimentConfigDef,
* experimentC: ExperimentConfigDef,
* }}
*/
let ExperimentsConfigDef;

/**
* @typedef {ExperimentConfigDef & {
* flavorType: string;
* rtvPrefixes: string[];
* command: string;
* }}
*/
let DistFlavorDef;

const argv = require('minimist')(process.argv.slice(2));
/** @type {ExperimentsConfigDef} */
const experimentsConfig = require('../../global-configs/experiments-config.json');
const fetch = require('node-fetch');
const fs = require('fs-extra');
Expand Down Expand Up @@ -104,24 +134,14 @@ function logSeparator_() {
log('---\n\n');
}

/**
* @typedef {{
* name: string,
* environment: string,
* issue: string,
* expiration_date_utc: string,
* define_experiment_constant: string,
* }}
*/
let ExperimentConfigDef;

/**
* Prepares output and temp directories.
*
* @param {string} outputDir full directory path to emplace artifacts in.
* @param {string} tempDir full directory path to temporary working directory.
*/
async function prepareEnvironment_(outputDir, tempDir) {
execOrDie('amp clean');
await fs.emptyDir(outputDir);
await fs.emptyDir(tempDir);
logSeparator_();
Expand All @@ -133,10 +153,10 @@ async function prepareEnvironment_(outputDir, tempDir) {
* The returned list of flavors will always contain the base flavor, and any
* defined experiments in ../../global-configs/experiments-config.json.
*
* @return {!Array<!Object>} list of AMP flavors to build.
* @return {!Array<!DistFlavorDef>} list of AMP flavors to build.
*/
function discoverDistFlavors_() {
const experimentConfigDefs = /** @type {[string, ExperimentConfigDef][]} */ (Object.entries(experimentsConfig));
const experimentConfigDefs = Object.entries(experimentsConfig);
const distFlavors = [
BASE_FLAVOR_CONFIG,
...experimentConfigDefs
Expand Down Expand Up @@ -164,7 +184,7 @@ function discoverDistFlavors_() {
'commands will be executed to compile each',
`${green('flavor')}:`
);
distFlavors.forEach(({flavorType, name, command}) => {
distFlavors.forEach(({command, flavorType, name}) => {
log('-', `(${green(flavorType)}, ${green(name)})`, cyan(command));
});

Expand All @@ -176,78 +196,72 @@ function discoverDistFlavors_() {
/**
* Compiles all AMP flavors sequentially.
*
* @param {!Array<!Object>} distFlavors list of AMP flavors to build.
* @param {string} flavorType AMP flavor to build.
* @param {string} command `amp` command to build the flavor.
* @param {string} tempDir full directory path to temporary working directory.
*/
async function compileDistFlavors_(distFlavors, tempDir) {
for (const {flavorType, command: baseCommand} of distFlavors) {
// TODO(danielrozenberg): remove undefined case when the release automation platform explicitly handles it.
const command =
argv.esm === undefined
? `${baseCommand} --esm && ${baseCommand}`
: argv.esm
? `${baseCommand} --esm`
: baseCommand;
log('Compiling flavor', green(flavorType), 'using', cyan(command));

execOrDie('amp clean');
execOrDie(command);

const flavorTempDistDir = path.join(tempDir, flavorType);
log('Moving build artifacts to', `${cyan(flavorTempDistDir)}...`);
await fs.ensureDir(flavorTempDistDir);

await Promise.all(
DIST_DIRS.map((distDir) =>
fs.move(distDir, path.join(flavorTempDistDir, distDir))
async function compileDistFlavors_(flavorType, command, tempDir) {
// TODO(danielrozenberg): remove undefined case when the release automation platform explicitly handles it.
if (argv.esm === undefined) {
command = `${command} --esm && ${command}`;
} else if (argv.esm) {
command += ' --esm';
}
log('Compiling flavor', green(flavorType), 'using', cyan(command));

execOrDie('amp clean --exclude release');
execOrDie(command);

const flavorTempDistDir = path.join(tempDir, flavorType);
log('Moving build artifacts to', `${cyan(flavorTempDistDir)}...`);
await fs.ensureDir(flavorTempDistDir);

await Promise.all(
DIST_DIRS.map((distDir) =>
fs.move(distDir, path.join(flavorTempDistDir, distDir))
)
);

log('Copying static files...');
const staticFilesPromises = [
// Directory-to-directory copy from the ./static sub-directory.
fs.copy(STATIC_FILES_DIR, path.join(flavorTempDistDir, 'dist')),
// Individual files to copy from the Git repository.
...STATIC_FILE_PATHS.map((staticFilePath) =>
fs.copy(
staticFilePath,
path.join(flavorTempDistDir, 'dist', path.basename(staticFilePath))
)
);
),
];
const postBuildMovesPromises = !argv.esm
? [
// Individual files to copy from the resulting build artifacts.
// This is only relevant for nomodule builds.
...Object.entries(POST_BUILD_MOVES).map(([from, to]) =>
fs.copy(
path.join(flavorTempDistDir, from),
path.join(flavorTempDistDir, to)
)
),
]
: [Promise.resolve()];
await Promise.all([...staticFilesPromises, ...postBuildMovesPromises]);

log('Copying static files...');
const staticFilesPromise = Promise.all([
// Directory-to-directory copy from the ./static sub-directory.
fs.copy(STATIC_FILES_DIR, path.join(flavorTempDistDir, 'dist')),
// Individual files to copy from the Git repository.
...STATIC_FILE_PATHS.map((staticFilePath) =>
fs.copy(
staticFilePath,
path.join(flavorTempDistDir, 'dist', path.basename(staticFilePath))
)
),
]);
/** @type {Promise} */
const postBuildMovesPromise = !argv.esm
? Promise.all([
// Individual files to copy from the resulting build artifacts.
// This is only relevant for nomodule builds.
...Object.entries(POST_BUILD_MOVES).map(([from, to]) =>
fs.copy(
path.join(flavorTempDistDir, from),
path.join(flavorTempDistDir, to)
)
),
])
: Promise.resolve();
await Promise.all([staticFilesPromise, postBuildMovesPromise]);

logSeparator_();
}
logSeparator_();
}

/**
* Fetches latest AMP service-worker package from the npm registry.
*
* @param {!Array<!Object>} distFlavors list of AMP flavors to build.
* @param {string} flavorType AMP flavor to build.
* @param {string} tempDir full directory path to temporary working directory.
*/
async function fetchAmpSw_(distFlavors, tempDir) {
async function fetchAmpSw_(flavorType, tempDir) {
const ampSwTempDir = path.join(tempDir, 'ampproject/amp-sw');
const distFlavorTypes = distFlavors.map(({flavorType}) => flavorType);
await Promise.all([
fs.ensureDir(ampSwTempDir),
...distFlavorTypes.map((flavorType) =>
fs.ensureDir(path.join(tempDir, flavorType, 'dist/sw'))
),
fs.ensureDir(path.join(tempDir, flavorType, 'dist/sw')),
]);

const ampSwNpmPackageResponse = await fetch(AMP_SW_NPM_PACKAGE_URL);
Expand All @@ -265,11 +279,7 @@ async function fetchAmpSw_(distFlavors, tempDir) {
tarWritableStream.on('end', resolve);
});

await Promise.all(
distFlavorTypes.map((flavorType) =>
fs.copy(ampSwTempDir, path.join(tempDir, flavorType, 'dist/sw'))
)
);
await fs.copy(ampSwTempDir, path.join(tempDir, flavorType, 'dist/sw'));

logSeparator_();
}
Expand All @@ -280,39 +290,34 @@ async function fetchAmpSw_(distFlavors, tempDir) {
* Each flavor translates to one or more RTV numbers, for a detailed explanation
* see spec/amp-framework-hosting.md.
*
* @param {!Array<!Object>} distFlavors list of AMP flavors to build.
* @param {string} flavorType AMP flavor to build.
* @param {!Array<string>} rtvPrefixes list of 2-digit RTV prefixes to generate.
* @param {string} tempDir full directory path to temporary working directory.
* @param {string} outputDir full directory path to emplace artifacts in.
*/
async function populateOrgCdn_(distFlavors, tempDir, outputDir) {
const rtvCopyingPromise = async (rtvPrefix, flavorType) => {
async function populateOrgCdn_(flavorType, rtvPrefixes, tempDir, outputDir) {
const rtvCopyingPromise = async (/** @type {string} */ rtvPrefix) => {
const rtvNumber = `${rtvPrefix}${VERSION}`;
const rtvPath = path.join(outputDir, 'org-cdn/rtv', rtvNumber);
await fs.ensureDir(rtvPath);
return fs.copy(path.join(tempDir, flavorType, 'dist'), rtvPath);
};

const rtvCopyingPromises = [];
for (const {flavorType, rtvPrefixes} of distFlavors) {
rtvCopyingPromises.push(
...rtvPrefixes.map((rtvPrefix) =>
rtvCopyingPromise(rtvPrefix, flavorType)
)
);
const rtvCopyingPromises = rtvPrefixes.map(rtvCopyingPromise);

// Special handling for INABOX experiments when compiling the base flavor.
// INABOX experiments need to have their control population be created from
// the base flavor.
if (flavorType == 'base') {
/** @type {[string, ExperimentConfigDef][]} */
(Object.entries(experimentsConfig))
// Special handling for INABOX experiments when compiling the base flavor.
// INABOX experiments need to have their control population be created from
// the base flavor.
if (flavorType == 'base') {
rtvCopyingPromises.push(
...Object.entries(experimentsConfig)
.filter(([, {environment}]) => environment == 'INABOX')
.forEach(([experimentFlavor]) => {
const rtvPrefix =
EXPERIMENTAL_RTV_PREFIXES['INABOX'][`${experimentFlavor}-control`];
rtvCopyingPromises.push(rtvCopyingPromise(rtvPrefix, 'base'));
});
}
.map(
([experimentFlavor]) =>
EXPERIMENTAL_RTV_PREFIXES['INABOX'][`${experimentFlavor}-control`]
)
.map(rtvCopyingPromise)
);
}
await Promise.all(rtvCopyingPromises);

Expand Down Expand Up @@ -447,7 +452,7 @@ async function release() {
await prepareEnvironment_(outputDir, tempDir);

log('Discovering release', `${green('flavors')}...`);
const distFlavors = await discoverDistFlavors_();
const distFlavors = discoverDistFlavors_();

if (argv.flavor && distFlavors.length == 0) {
log('Flavor', cyan(argv.flavor), 'is inactive. Quitting...');
Expand All @@ -457,13 +462,16 @@ async function release() {
if (!argv.flavor) {
log('Compiling all', `${green('flavors')}...`);
}
await compileDistFlavors_(distFlavors, tempDir);

log('Fetching npm package', `${cyan('@ampproject/amp-sw')}...`);
await fetchAmpSw_(distFlavors, tempDir);
for (const {command, flavorType, rtvPrefixes} of distFlavors) {
await compileDistFlavors_(flavorType, command, tempDir);

log('Fetching npm package', `${cyan('@ampproject/amp-sw')}...`);
await fetchAmpSw_(flavorType, tempDir);

log('Copying from temporary directory to', cyan('org-cdn'));
await populateOrgCdn_(distFlavors, tempDir, outputDir);
log('Copying from temporary directory to', cyan('org-cdn'));
await populateOrgCdn_(flavorType, rtvPrefixes, tempDir, outputDir);
}

log('Generating', cyan('files.txt'), 'files in', cyan('org-cdn/rtv/*'));
await generateFileListing_(outputDir);
Expand Down

0 comments on commit c5a5cf5

Please sign in to comment.