🏗 Add conditional dist job for push builds (#20930)

.travis.yml

@@ -57,6 +57,11 @@ jobs:
           packages:
             - protobuf-compiler
             - python-protobuf
+    - stage: build
+      name: "Dist"
+      if: type != pull_request
+      script:
+        - node build-system/pr-check/dist.js      
     - stage: test
       name: "Dist, Bundle Size, Single Pass Tests"
       addons:

build-system/pr-check/build.js

@@ -21,6 +21,7 @@
  * This is run during the CI stage = build; job = build.
  */
 
+const colors = require('ansi-colors');
 const {
   areValidBuildTargets,
   determineBuildTargets} = require('./build-targets');
@@ -35,6 +36,7 @@ const {
   uploadBuildOutput} = require('./utils');
 const {isTravisPullRequestBuild} = require('../travis');
 const FILENAME = 'build.js';
+const FILELOGPREFIX = colors.bold(colors.yellow(`${FILENAME}:`));
 const timedExecOrDie =
   (cmd, unusedFileName) => timedExecOrDieBase(cmd, FILENAME);
 
@@ -69,8 +71,9 @@ function main() {
       timedExecOrDie('gulp build --fortesting');
       uploadBuildOutput(FILENAME);
     } else {
-      console.log('Skipping build job because this commit does not affect ' +
-          'the runtime, build system, test files, or the dev dashboard');
+      console.log(`${FILELOGPREFIX} Skipping ` + colors.cyan('Build ') +
+          'because this commit does not affect the runtime, ' +
+          'build system, test files, or the dev dashboard');
     }
   }
 

build-system/pr-check/dist-tests.js

@@ -23,6 +23,7 @@
 
 const colors = require('ansi-colors');
 const {
+  downloadDistOutput,
   printChangeSummary,
   startTimer,
   stopTimer,
@@ -40,15 +41,14 @@ function runSinglePassTest_() {
   timedExecOrDie('gulp dist --fortesting --single_pass --psuedonames');
   timedExecOrDie('gulp test --integration ' +
       '--nobuild --compiled --single_pass --headless');
-  timedExecOrDie('gulp clean');
 }
 
 function main() {
   const startTime = startTimer(FILENAME, FILENAME);
   const buildTargets = determineBuildTargets();
 
   if (!isTravisPullRequestBuild()) {
-    timedExecOrDie('gulp dist --fortesting --noextensions');
+    downloadDistOutput();
     timedExecOrDie('gulp bundle-size --on_push_build');
     runSinglePassTest_();
   } else {
@@ -72,9 +72,10 @@ function main() {
     }
 
     if (!ranTests) {
-      console.log(`${FILELOGPREFIX} Skipping dist tests because this commit ` +
-        'does not affect the runtime, build system, or ' +
-        'integration test files.');
+      console.log(`${FILELOGPREFIX} Skipping ` +
+          colors.cyan('Dist, Bundle Size, Single Pass Tests ') +
+          'because this commit does not affect the runtime, build system, ' +
+          'or integration test files.');
     }
   }
 

build-system/pr-check/dist.js

@@ -0,0 +1,53 @@
+/**
+ * Copyright 2019 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+/**
+ * @fileoverview
+ * This script builds the AMP runtime for production.
+ * This is run during the CI stage = build; job = dist.
+ */
+
+const colors = require('ansi-colors');
+const {
+  printChangeSummary,
+  startTimer,
+  stopTimer,
+  timedExecOrDie: timedExecOrDieBase,
+  uploadDistOutput} = require('./utils');
+const {isTravisPullRequestBuild} = require('../travis');
+const FILENAME = 'dist.js';
+const FILELOGPREFIX = colors.bold(colors.yellow(`${FILENAME}:`));
+const timedExecOrDie =
+  (cmd, unusedFileName) => timedExecOrDieBase(cmd, FILENAME);
+
+function main() {
+  const startTime = startTimer(FILENAME, FILENAME);
+
+  if (!isTravisPullRequestBuild()) {
+    timedExecOrDie('gulp dist --fortesting');
+    uploadDistOutput(FILENAME);
+  } else {
+    printChangeSummary();
+    console.log(`${FILELOGPREFIX} Skipping ` + colors.cyan('Dist ') +
+        'because this is a PR build');
+  }
+
+  stopTimer(FILENAME, FILENAME, startTime);
+  return 0;
+}
+
+process.exit(main());

build-system/pr-check/local-tests.js

@@ -56,8 +56,8 @@ function main() {
           buildTargets.has('INTEGRATION_TEST') ||
           buildTargets.has('DEV_DASHBOARD'))) {
       console.log(
-          `${FILELOGPREFIX} Skipping unit and integration tests because ` +
-          'this commit not affect the runtime, build system, ' +
+          `${FILELOGPREFIX} Skipping ` + colors.cyan('Local Tests ') +
+          'because this commit not affect the runtime, build system, ' +
           'unit test files, integration test files, or the dev dashboard.');
       stopTimer(FILENAME, FILENAME, startTime);
       return 0;

build-system/pr-check/remote-tests.js

@@ -24,6 +24,7 @@
 const colors = require('ansi-colors');
 const {
   downloadBuildOutput,
+  downloadDistOutput,
   printChangeSummary,
   startTimer,
   stopTimer,
@@ -43,13 +44,10 @@ function main() {
   const buildTargets = determineBuildTargets();
 
   if (!isTravisPullRequestBuild()) {
+    downloadDistOutput(FILENAME);
     startSauceConnect(FILENAME);
-    downloadBuildOutput(FILENAME);
-
     timedExecOrDie('gulp test --unit --nobuild --saucelabs_lite');
-    timedExecOrDie('gulp dist --fortesting');
     timedExecOrDie('gulp test --integration --nobuild --compiled --saucelabs');
-
     stopSauceConnect(FILENAME);
   } else {
     printChangeSummary(FILENAME);
@@ -58,14 +56,15 @@ function main() {
           buildTargets.has('UNIT_TEST') ||
           buildTargets.has('INTEGRATION_TEST'))) {
       console.log(
-          `${FILELOGPREFIX} Skipping Sauce Labs unit and integration tests ` +
-          'because this commit does not affect the runtime, build system, ' +
-          'or integration test files.');
+          `${FILELOGPREFIX} Skipping ` +
+          colors.cyan('Remote (Sauce Labs) Tests ') +
+          'because this commit does not affect the runtime, ' +
+          'build system, or integration test files.');
       stopTimer(FILENAME, FILENAME, startTime);
       return 0;
     }
-    startSauceConnect(FILENAME);
     downloadBuildOutput(FILENAME);
+    startSauceConnect(FILENAME);
 
     if (buildTargets.has('RUNTIME') ||
         buildTargets.has('BUILD_SYSTEM') ||

build-system/pr-check/utils.js

@@ -28,9 +28,11 @@ const {execOrDie, exec, getStdout} = require('../exec');
 const {isTravisBuild, travisBuildNumber, travisPullRequestSha} = require('../travis');
 
 const BUILD_OUTPUT_FILE =
-  isTravisBuild() ? `amp_build_${travisBuildNumber()}.zip` : '';
-const BUILD_OUTPUT_DIRS = 'build/ dist/ dist.3p/ EXTENSIONS_CSS_MAP';
-const BUILD_OUTPUT_STORAGE_LOCATION = 'gs://amp-travis-builds';
+    isTravisBuild() ? `amp_build_${travisBuildNumber()}.zip` : '';
+const DIST_OUTPUT_FILE =
+    isTravisBuild() ? `amp_dist_${travisBuildNumber()}.zip` : '';
+const OUTPUT_DIRS = 'build/ dist/ dist.3p/ EXTENSIONS_CSS_MAP';
+const OUTPUT_STORAGE_LOCATION = 'gs://amp-travis-builds';
 
 /**
  * Prints a summary of files changed by, and commits included in the PR.
@@ -143,58 +145,96 @@ function timedExecOrDie(cmd, fileName = 'utils.js') {
 }
 
 /**
- * Downloads build output from storage
+ * Download output helper
  * @param {string} functionName
+ * @param {string} outputFileName
+ * @private
  */
-function downloadBuildOutput(functionName) {
+function downloadOutput_(functionName, outputFileName) {
   const fileLogPrefix = colors.bold(colors.yellow(`${functionName}:`));
   const buildOutputDownloadUrl =
-    `${BUILD_OUTPUT_STORAGE_LOCATION}/${BUILD_OUTPUT_FILE}`;
+    `${OUTPUT_STORAGE_LOCATION}/${outputFileName}`;
 
   console.log(
       `${fileLogPrefix} Downloading build output from ` +
       colors.cyan(buildOutputDownloadUrl) + '...');
   exec('echo travis_fold:start:download_results && echo');
-  execOrDie(`gsutil cp ${buildOutputDownloadUrl} ${BUILD_OUTPUT_FILE}`);
+  execOrDie(`gsutil cp ${buildOutputDownloadUrl} ${outputFileName}`);
   exec('echo travis_fold:end:download_results');
 
   console.log(
-      `${fileLogPrefix} Extracting ` + colors.cyan(BUILD_OUTPUT_FILE) + '...');
+      `${fileLogPrefix} Extracting ` + colors.cyan(outputFileName) + '...');
   exec('echo travis_fold:start:unzip_results && echo');
-  execOrDie(`unzip -o ${BUILD_OUTPUT_FILE}`);
+  execOrDie(`unzip -o ${outputFileName}`);
   exec('echo travis_fold:end:unzip_results');
 
   console.log(fileLogPrefix, 'Verifying extracted files...');
   exec('echo travis_fold:start:verify_unzip_results && echo');
-  execOrDie(`ls -la ${BUILD_OUTPUT_DIRS}`);
+  execOrDie(`ls -la ${OUTPUT_DIRS}`);
   exec('echo travis_fold:end:verify_unzip_results');
 }
 
 /**
- * Zips and uploads the build output to a remote storage location
+ * Upload output helper
  * @param {string} functionName
+ * @param {string} outputFileName
+ * @private
  */
-function uploadBuildOutput(functionName) {
+function uploadOutput_(functionName, outputFileName) {
   const fileLogPrefix = colors.bold(colors.yellow(`${functionName}:`));
 
   console.log(
-      `\n${fileLogPrefix} Compressing ` + colors.cyan(BUILD_OUTPUT_DIRS) +
-      ' into ' + colors.cyan(BUILD_OUTPUT_FILE) + '...');
+      `\n${fileLogPrefix} Compressing ` +
+      colors.cyan(OUTPUT_DIRS.split(' ').join(', ')) +
+      ' into ' + colors.cyan(outputFileName) + '...');
   exec('echo travis_fold:start:zip_results && echo');
-  execOrDie(`zip -r ${BUILD_OUTPUT_FILE} ${BUILD_OUTPUT_DIRS}`);
+  execOrDie(`zip -r ${outputFileName} ${OUTPUT_DIRS}`);
   exec('echo travis_fold:end:zip_results');
 
   console.log(
-      `${fileLogPrefix} Uploading ` + colors.cyan(BUILD_OUTPUT_FILE) + ' to ' +
-      colors.cyan(BUILD_OUTPUT_STORAGE_LOCATION) + '...');
+      `${fileLogPrefix} Uploading ` + colors.cyan(outputFileName) + ' to ' +
+      colors.cyan(OUTPUT_STORAGE_LOCATION) + '...');
   exec('echo travis_fold:start:upload_results && echo');
-  execOrDie(`gsutil -m cp -r ${BUILD_OUTPUT_FILE} ` +
-      `${BUILD_OUTPUT_STORAGE_LOCATION}`);
+  execOrDie(`gsutil -m cp -r ${outputFileName} ` +
+      `${OUTPUT_STORAGE_LOCATION}`);
   exec('echo travis_fold:end:upload_results');
 }
 
+/**
+ * Downloads and unzips build output from storage
+ * @param {string} functionName
+ */
+function downloadBuildOutput(functionName) {
+  downloadOutput_(functionName, BUILD_OUTPUT_FILE);
+}
+
+/**
+ * Downloads and unzips dist output from storage
+ * @param {string} functionName
+ */
+function downloadDistOutput(functionName) {
+  downloadOutput_(functionName, DIST_OUTPUT_FILE);
+}
+
+/**
+ * Zips and uploads the build output to a remote storage location
+ * @param {string} functionName
+ */
+function uploadBuildOutput(functionName) {
+  uploadOutput_(functionName, BUILD_OUTPUT_FILE);
+}
+
+/**
+ * Zips and uploads the dist output to a remote storage location
+ * @param {string} functionName
+ */
+function uploadDistOutput(functionName) {
+  uploadOutput_(functionName, DIST_OUTPUT_FILE);
+}
+
 module.exports = {
   downloadBuildOutput,
+  downloadDistOutput,
   printChangeSummary,
   startTimer,
   stopTimer,
@@ -203,4 +243,5 @@ module.exports = {
   timedExec,
   timedExecOrDie,
   uploadBuildOutput,
+  uploadDistOutput,
 };

build-system/pr-check/validator-tests.js

@@ -50,8 +50,9 @@ function main() {
       timedExecOrDie('gulp validator-webui');
     } else {
       console.log(
-          `${FILELOGPREFIX} Skipping validator job because this commit does ` +
-          'not affect the validator or validator web UI.');
+          `${FILELOGPREFIX} Skipping ` + colors.cyan('Validator Tests ') +
+          'because this commit does not affect ' +
+          'the validator or validator web UI.');
     }
   }
 

build-system/pr-check/visual-diff-tests.js

@@ -59,9 +59,10 @@ function main() {
     } else {
       timedExecOrDie('gulp visual-diff --nobuild --empty');
       console.log(
-          `${FILELOGPREFIX} Skipping visual diff tests because this commit ` +
-          'does not affect the runtime, build system, integration ' +
-          'test files, visual diff test files, or flag config files.');
+          `${FILELOGPREFIX} Skipping ` + colors.cyan('Visual Diff Tests ') +
+          'because this commit does not affect the ' +
+          'runtime, build system, integration test files, ' +
+          'visual diff test files, or flag config files.');
     }
   }
 

build-system/tasks/presubmit-checks.js

@@ -107,6 +107,7 @@ const forbiddenTerms = {
       'build-system/pr-check/build.js',
       'build-system/pr-check/build-targets.js',
       'build-system/pr-check/checks.js',
+      'build-system/pr-check/dist.js',
       'build-system/pr-check/dist-tests.js',
       'build-system/pr-check/local-tests.js',
       'build-system/pr-check/remote-tests.js',