Travis imports

lib/deps.js

@@ -9,10 +9,9 @@ const Tmp = require('tmp');
 const Package = require('./package');
 const Utils = require('./utils');
 
-const internals = {};
-
-
-internals.log = Debug('detect-node-support');
+const internals = {
+    log: Debug('detect-node-support')
+};
 
 
 internals.resolve = async ({ packageJson, lockfile }, options) => {

lib/loader.js

@@ -11,6 +11,7 @@ const Wreck = require('@hapi/wreck');
 const Utils = require('./utils');
 
 const internals = {
+    cache: new Map(),
     log: Debug('detect-node-support:loader'),
     error: Debug('detect-node-support:error')
 };
@@ -95,9 +96,18 @@ internals.createRepositoryLoader = (repository) => {
             const url = `https://raw.githubusercontent.com/${parsedRepository.full_name}/HEAD/${filename}`;
             internals.log('Loading: %s', url);
 
+            if (options === undefined && internals.cache.has(url)) {
+                internals.log('From cache: %s', url);
+                return internals.cache.get(url);
+            }
+
             try {
                 const { payload } = await Wreck.get(url, options);
 
+                if (options === undefined) {
+                    internals.cache.set(url, payload);
+                }
+
                 internals.log('Loaded: %s', url);
                 return payload;
             }
@@ -164,3 +174,9 @@ exports.create = ({ path, repository, packageName }) => {
 
     return internals.createPathLoader(path);
 };
+
+
+exports.clearCache = () => {
+
+    internals.cache = new Map();
+};

lib/travis/imports.js

@@ -0,0 +1,102 @@
+'use strict';
+
+const Yaml = require('js-yaml');
+
+const Loader = require('../loader');
+const Utils = require('../utils');
+
+const TravisMerge = require('./merge');
+
+
+const internals = {
+    validMergeModes: new Set(['deep_merge_append', 'deep_merge_prepend', 'deep_merge', 'merge'])
+};
+
+
+internals.normalizeImports = (travisYaml, { relativeTo, breadcrumb }) => {
+
+    const context = relativeTo ? relativeTo.source : '.travis.yml';
+
+    return Utils.toArray(travisYaml.import)
+        .map((entry) => {
+
+            if (typeof entry === 'string') {
+                entry = { source: entry };
+            }
+
+            const original = entry.source;
+
+            if (entry.source.startsWith('./')) {
+                entry.source = entry.source.substring(2);
+
+                if (relativeTo) {
+                    const relativeParts = relativeTo.source.split('/');
+                    relativeParts.pop();
+                    relativeParts.push(entry.source);
+                    entry.source = relativeParts.join('/');
+                }
+            }
+
+            if (!entry.mode) {
+                entry.mode = 'deep_merge_append';
+            }
+
+            if (!internals.validMergeModes.has(entry.mode)) {
+                throw new Error(`Invalid merge mode for ${original} in ${context}: ${entry.mode}`);
+            }
+
+            if (original.includes('@')) {
+                throw new Error(`Importing at commitish unsupported in ${context}: ${original}`);
+            }
+
+            const alreadyImported = breadcrumb.indexOf(entry.source);
+            if (alreadyImported >= 0) {
+                throw new Error(`Circular dependency ${entry.source} requested by ${context} (already imported at ${breadcrumb[alreadyImported - 1]})`);
+            }
+
+            return entry;
+        })
+        .filter((entry) => !entry.if); // @todo: log a warning
+};
+
+
+internals.loadSource = async (source, { loadFile }) => {
+
+    let path = source;
+
+    if (source.includes(':')) {
+        const [repository, fileName] = source.split(':');
+        const loader = await Loader.create({ repository: `https://github.com/${repository}` });
+
+        path = fileName;
+        loadFile = loader.loadFile;
+    }
+
+    return loadFile(path);
+};
+
+
+exports.apply = async (yaml, { loadFile, relativeTo, breadcrumb = ['.travis.yml'] }) => {
+
+    if (!yaml.import) {
+        return;
+    }
+
+    const imports = internals.normalizeImports(yaml, { relativeTo, breadcrumb });
+
+    for (const entry of imports) {
+
+        const buffer = await internals.loadSource(entry.source, { loadFile });
+
+        const imported = Yaml.safeLoad(buffer, {
+            schema: Yaml.FAILSAFE_SCHEMA,
+            json: true
+        });
+
+        await exports.apply(imported, { loadFile, relativeTo: entry, breadcrumb: [...breadcrumb, entry.source] });
+
+        delete imported.import;
+
+        TravisMerge[entry.mode](yaml, imported);
+    }
+};

lib/travis/index.js

@@ -3,39 +3,35 @@
 const Nv = require('@pkgjs/nv');
 const Yaml = require('js-yaml');
 
+const TravisImports = require('./imports');
+const Utils = require('../utils');
+
 
 const internals = {};
 
+
 internals.nodeAliases = {
     latest: 'active',
     node: 'active',
     stable: 'active'
 };
 
 
-internals.toArray = (v) => {
-
-    if (v === undefined) {
-        return [];
-    }
-
-    return Array.isArray(v) ? v : [v];
-};
-
+internals.scan = async (travisYaml, options) => {
 
-internals.scan = async (travisYaml) => {
+    await TravisImports.apply(travisYaml, options);
 
     const rawSet = new Set();
 
-    for (const v of internals.toArray(travisYaml.node_js)) {
+    for (const v of Utils.toArray(travisYaml.node_js)) {
         rawSet.add(v);
     }
 
     if (travisYaml.env) {
 
-        for (const env of internals.toArray(travisYaml.env.matrix)) {
+        for (const env of Utils.toArray(travisYaml.env.matrix)) {
 
-            const matches = env.match(/(?:NODEJS_VER|TRAVIS_NODE_VERSION|NODE_VER)="?(node\/)?(?<version>[\w./*]+)"?/); /* hack syntax highlighter 🤦‍♂️ */
+            const matches = env.match(/(?:NODEJS_VER|TRAVIS_NODE_VERSION|NODE_VER)="?(node\/)?(?<version>[\w./*]+)"?/);
 
             if (matches) {
                 rawSet.add(matches.groups.version);
@@ -45,7 +41,7 @@ internals.scan = async (travisYaml) => {
 
     if (travisYaml.matrix) {
 
-        for (const include of internals.toArray(travisYaml.matrix.include)) {
+        for (const include of Utils.toArray(travisYaml.matrix.include)) {
 
             if (include.node_js) {
                 rawSet.add(include.node_js);
@@ -96,6 +92,6 @@ exports.detect = async ({ loadFile }) => {
     });
 
     return {
-        travis: await internals.scan(travisYaml)
+        travis: await internals.scan(travisYaml, { loadFile })
     };
 };

lib/travis/merge.js

@@ -0,0 +1,65 @@
+'use strict';
+
+// ref: https://github.com/travis-ci/travis-yml/blob/bf82881491134c72a64778f9664a8dd3f97158e7/lib/travis/yml/support/merge.rb
+
+const internals = {};
+
+
+internals.isObject = (arg) => typeof arg === 'object' && !Array.isArray(arg);
+
+
+exports.deep_merge_append = (left, right) => {
+
+    for (const key in right) {
+
+        if (internals.isObject(left[key]) && internals.isObject(right[key])) {
+            exports.deep_merge_append(left[key], right[key]);
+            continue;
+        }
+
+        if (Array.isArray(left[key]) && Array.isArray(right[key])) {
+            left[key].push(...right[key]);
+            continue;
+        }
+
+        left[key] = right[key];
+    }
+};
+
+exports.deep_merge_prepend = (left, right) => {
+
+    for (const key in right) {
+
+        if (internals.isObject(left[key]) && internals.isObject(right[key])) {
+            exports.deep_merge_prepend(left[key], right[key]);
+            continue;
+        }
+
+        if (Array.isArray(left[key]) && Array.isArray(right[key])) {
+            left[key].unshift(...right[key]);
+            continue;
+        }
+
+        left[key] = right[key];
+    }
+};
+
+exports.deep_merge = (left, right) => {
+
+    for (const key in right) {
+
+        if (internals.isObject(left[key]) && internals.isObject(right[key])) {
+            exports.deep_merge(left[key], right[key]);
+            continue;
+        }
+
+        left[key] = right[key];
+    }
+};
+
+exports.merge = (left, right) => {
+
+    for (const key in right) {
+        left[key] = right[key];
+    }
+};

lib/utils.js

@@ -22,3 +22,13 @@ exports.getErrorMessage = (error) => {
 
     return null;
 };
+
+
+exports.toArray = (v) => {
+
+    if (v === undefined) {
+        return [];
+    }
+
+    return Array.isArray(v) ? v : [v];
+};

test/fixtures/index.js

@@ -7,6 +7,7 @@ const SimpleGit = require('simple-git/promise');
 const Sinon = require('sinon');
 const Tmp = require('tmp');
 
+const Loader = require('../../lib/loader');
 const Utils = require('../../lib/utils');
 
 
@@ -28,6 +29,8 @@ module.exports = class TestContext {
 
     cleanup() {
 
+        Loader.clearCache();
+
         Sinon.restore();
 
         this._cleanup.forEach((cleanup) => cleanup());
@@ -75,7 +78,7 @@ module.exports = class TestContext {
         });
     }
 
-    async setupRepoFolder({ travisYml, packageJson, npmShrinkwrapJson, packageLockJson, git = true } = {}) {
+    async setupRepoFolder({ travisYml, partials, packageJson, npmShrinkwrapJson, packageLockJson, git = true } = {}) {
 
         const tmpObj = Tmp.dirSync({ unsafeCleanup: true });
 
@@ -87,6 +90,22 @@ module.exports = class TestContext {
             Fs.copyFileSync(Path.join(__dirname, 'travis-ymls', travisYml), Path.join(this.path, '.travis.yml'));
         }
 
+        if (partials) {
+            Fs.mkdirSync(Path.join(this.path, 'partials'));
+            const partialYmls = [
+                'circular.yml',
+                'commitish.yml',
+                'indirect-node-14.yml',
+                'merge-invalid.yml',
+                'node-10.yml',
+                'node-12.yml',
+                'node-14.yml'
+            ];
+            for (const fn of partialYmls) {
+                Fs.copyFileSync(Path.join(__dirname, 'travis-ymls', 'testing-imports', 'partials', fn), Path.join(this.path, 'partials', fn));
+            }
+        }
+
         if (packageJson !== false) {
             Fs.writeFileSync(Path.join(this.path, 'package.json'), JSON.stringify(packageJson || {
                 name: 'test-module',

test/fixtures/travis-ymls/testing-imports/another-repo.yml

@@ -0,0 +1,4 @@
+language: node_js
+import:
+  - source: pkgjs/detect-node-support:test/fixtures/travis-ymls/testing-imports/partials/indirect-node-14.yml
+  - source: pkgjs/detect-node-support:test/fixtures/travis-ymls/testing-imports/partials/node-14.yml # cache hit

test/fixtures/travis-ymls/testing-imports/circular.yml

@@ -0,0 +1,3 @@
+language: node_js
+import:
+  - source: partials/circular.yml

test/fixtures/travis-ymls/testing-imports/commitish.yml

@@ -0,0 +1,3 @@
+language: node_js
+import:
+  - source: partials/commitish.yml

test/fixtures/travis-ymls/testing-imports/conditional.yml

@@ -0,0 +1,4 @@
+language: node_js
+import:
+  - source: partials/indirect-node-14.yml
+    if: branch = master

test/fixtures/travis-ymls/testing-imports/indirect-dot-slash.yml

@@ -0,0 +1,3 @@
+language: node_js
+import:
+  - source: ./partials/indirect-node-14.yml

test/fixtures/travis-ymls/testing-imports/indirect.yml

@@ -0,0 +1,3 @@
+language: node_js
+import:
+  - source: partials/indirect-node-14.yml

test/fixtures/travis-ymls/testing-imports/merge-deep-prepend-append.yml

@@ -0,0 +1,9 @@
+language: node_js
+node_js:
+  - "8"
+import:
+  - source: partials/node-14.yml # default merge: deep_merge_append
+  - source: partials/node-12.yml
+    mode: deep_merge_prepend
+  - source: partials/node-10.yml
+    mode: deep_merge_append

test/fixtures/travis-ymls/testing-imports/merge-deep.yml

@@ -0,0 +1,7 @@
+language: node_js
+node_js:
+  - "8"
+import:
+  - source: partials/node-14.yml # default merge: deep_merge_append
+  - source: partials/node-12.yml
+    mode: deep_merge

test/fixtures/travis-ymls/testing-imports/merge-invalid.yml

@@ -0,0 +1,5 @@
+language: node_js
+node_js:
+  - "8"
+import:
+  - source: partials/merge-invalid.yml