Copy integration files to a new test-bin directory in preparation for…

… postHTML transformation (#28864)

* move buildNewServer to its own file

* temp

* temp

* temp

* normalize all script paths to cdn

* fix cupcake file

* temp

* add new test-bin directory for transformed fixtures

* temp

* revert all test files

* minor clean up

* fix lint issues

* revert changes to transformer sequence

* move call to buildTransformedHtml after `shouldNotRun` call

* get all tests green by switching the "served" path back to the original

* fix lint issues

* move typescript compile to server folder

* apply recs

.gitignore

@@ -7,6 +7,7 @@ c
 /dist
 dist.3p
 dist.tools
+test-bin
 examples.min
 node_modules
 npm-debug.log

build-system/server/typescript-compile.js

@@ -0,0 +1,47 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const log = require('fancy-log');
+const {cyan, green} = require('ansi-colors');
+const {exec} = require('../common/exec');
+
+const SERVER_TRANSFORM_PATH = 'build-system/server/new-server/transforms';
+
+// Used by new server implementation
+const typescriptBinary = './node_modules/typescript/bin/tsc';
+
+/**
+ * Builds the new server by converting typescript transforms to JS
+ */
+function buildNewServer() {
+  const buildCmd = `${typescriptBinary} -p ${SERVER_TRANSFORM_PATH}/tsconfig.json`;
+  log(
+    green('Building'),
+    cyan('AMP Dev Server'),
+    green('at'),
+    cyan(`${SERVER_TRANSFORM_PATH}/dist`) + green('...')
+  );
+  const result = exec(buildCmd, {'stdio': ['inherit', 'inherit', 'pipe']});
+  if (result.status != 0) {
+    const err = new Error('Could not build AMP Dev Server');
+    err.showStack = false;
+    throw err;
+  }
+}
+
+module.exports = {
+  buildNewServer,
+  SERVER_TRANSFORM_PATH,
+};

build-system/tasks/clean.js

@@ -33,6 +33,7 @@ async function clean() {
     'build-system/runner/build',
     'build-system/runner/dist',
     'build-system/server/new-server/transforms/dist',
+    'test-bin',
   ]);
 }
 

build-system/tasks/integration.js

@@ -16,6 +16,9 @@
 'using strict';
 
 const argv = require('minimist')(process.argv.slice(2));
+const fs = require('fs-extra');
+const globby = require('globby');
+const log = require('fancy-log');
 const {
   maybePrintArgvMessages,
   shouldNotRun,
@@ -25,6 +28,7 @@ const {
   RuntimeTestConfig,
 } = require('./runtime-test/runtime-test-base');
 const {buildRuntime} = require('../common/utils');
+const {cyan, green} = require('ansi-colors');
 
 class Runner extends RuntimeTestRunner {
   constructor(config) {
@@ -40,11 +44,32 @@ class Runner extends RuntimeTestRunner {
   }
 }
 
+async function buildTransformedHtml() {
+  const filePaths = await globby('./test/fixtures/**/*.html');
+  let normalizedFilePath;
+  try {
+    log(
+      green('Copying integration test files to'),
+      cyan('test-bin/') + green('...')
+    );
+    for (const filePath of filePaths) {
+      await fs.copySync(filePath, `./test-bin/${filePath}`);
+    }
+  } catch (e) {
+    console./*OK*/ log(
+      `${normalizedFilePath} could not be transformed by the postHTML ` +
+        `pipeline.\n${e.message}`
+    );
+  }
+}
+
 async function integration() {
   if (shouldNotRun()) {
     return;
   }
 
+  await buildTransformedHtml();
+
   maybePrintArgvMessages();
 
   const config = new RuntimeTestConfig('integration');

build-system/tasks/karma.conf.js

@@ -85,13 +85,20 @@ module.exports = {
   ],
 
   preprocessors: {
-    './test/fixtures/*.html': ['html2js'],
+    // `test-bin` is the output directory of the postHTML transformation.
+    './test-bin/test/fixtures/*.html': ['html2js'],
     './test/**/*.js': ['browserify'],
     './ads/**/test/test-*.js': ['browserify'],
     './extensions/**/test/**/*.js': ['browserify'],
     './testing/**/*.js': ['browserify'],
   },
 
+  html2JsPreprocessor: {
+    // Strip the test-bin/ prefix for the transformer destination so that the
+    // change is transparent for users of the path.
+    stripPrefix: 'test-bin/',
+  },
+
   // TODO(rsimha, #15510): Sauce labs on Safari doesn't reliably support
   // 'localhost' addresses. See #14848 for more info.
   // Details: https://support.saucelabs.com/hc/en-us/articles/115010079868

build-system/tasks/serve.js

@@ -24,6 +24,10 @@ const minimist = require('minimist');
 const morgan = require('morgan');
 const path = require('path');
 const watch = require('gulp-watch');
+const {
+  buildNewServer,
+  SERVER_TRANSFORM_PATH,
+} = require('../server/typescript-compile');
 const {
   lazyBuildExtensions,
   lazyBuildJs,
@@ -33,24 +37,19 @@ const {
 const {createCtrlcHandler} = require('../common/ctrlcHandler');
 const {cyan, green, red} = require('ansi-colors');
 const {distNailgunPort, stopNailgunServer} = require('./nailgun');
-const {exec} = require('../common/exec');
 const {logServeMode, setServeMode} = require('../server/app-utils');
 const {watchDebounceDelay} = require('./helpers');
 
 const argv = minimist(process.argv.slice(2), {string: ['rtv']});
 
-// Used by new server implementation
-const typescriptBinary = './node_modules/typescript/bin/tsc';
-const transformsPath = 'build-system/server/new-server/transforms';
-
 // Used for logging.
 let url = null;
 let quiet = !!argv.quiet;
 
 // Used for live reload.
 const serverFiles = globby.sync([
   'build-system/server/**',
-  `!${transformsPath}/dist/**`,
+  `!${SERVER_TRANSFORM_PATH}/dist/**`,
 ]);
 
 // Used to enable / disable lazy building.
@@ -117,25 +116,6 @@ async function startServer(
   logServeMode();
 }
 
-/**
- * Builds the new server by converting typescript transforms to JS
- */
-function buildNewServer() {
-  const buildCmd = `${typescriptBinary} -p ${transformsPath}/tsconfig.json`;
-  log(
-    green('Building'),
-    cyan('AMP Dev Server'),
-    green('at'),
-    cyan(`${transformsPath}/dist`) + green('...')
-  );
-  const result = exec(buildCmd, {'stdio': ['inherit', 'inherit', 'pipe']});
-  if (result.status != 0) {
-    const err = new Error('Could not build AMP Dev Server');
-    err.showStack = false;
-    throw err;
-  }
-}
-
 /**
  * Clears server files from the require cache to allow for in-process server
  * live-reload.
@@ -212,7 +192,6 @@ async function doServe(lazyBuild = false) {
 }
 
 module.exports = {
-  buildNewServer,
   serve,
   doServe,
   startServer,

build-system/tasks/server-tests.js

@@ -22,7 +22,7 @@ const log = require('fancy-log');
 const path = require('path');
 const posthtml = require('posthtml');
 const through = require('through2');
-const {buildNewServer} = require('../tasks/serve');
+const {buildNewServer} = require('../server/typescript-compile');
 const {cyan, green, red} = require('ansi-colors');
 const {isTravisBuild} = require('../common/travis');
 

build-system/test-configs/config.js

@@ -18,7 +18,7 @@
 const initTestsPath = ['test/_init_tests.js'];
 
 const fixturesExamplesPaths = [
-  'test/fixtures/*.html',
+  'test-bin/test/fixtures/*.html',
   {
     pattern: 'test/fixtures/served/*.html',
     included: false,