feat: test plugin dep-trees as graphs via new /test-dep-graph API

package.json

@@ -27,7 +27,7 @@
     "eslint": "eslint -c .eslintrc src",
     "tslint": "tslint --project tsconfig.json --format stylish --exclude **/src/**/*.js",
     "prepare": "npm run build",
-    "tap": "tap test/*.test.* -Rspec --timeout=180 --node-path ts-node --test-file-pattern '/\\.[tj]s$/'",
+    "tap": "tap test/*.test.* -Rspec --timeout=300 --node-path ts-node --test-file-pattern '/\\.[tj]s$/'",
     "test": "npm run test-common && npm run tap",
     "test-common": "npm run check-tests && npm run lint && node --require ts-node/register src/cli test --org=snyk",
     "lint": "npm run eslint && npm run tslint",
@@ -42,6 +42,8 @@
   "author": "snyk.io",
   "license": "Apache-2.0",
   "dependencies": {
+    "@snyk/dep-graph": "1.1.1",
+    "@snyk/gemfile": "1.1.0",
     "abbrev": "^1.1.1",
     "ansi-escapes": "^3.1.0",
     "chalk": "^2.4.1",

src/lib/plugins/rubygems/gemfile-lock-to-dependencies.js

@@ -0,0 +1,49 @@
+const gemfile = require('@snyk/gemfile');
+
+module.exports = gemfileLockToDependencies;
+
+const detectCycles = (dep, chain) => {
+  if (chain.indexOf(dep) >= 0) {
+    const error = Error('Cyclic dependency detected in lockfile');
+    const UNPROCESSABLE_ENTITY = 422;
+    error.code = UNPROCESSABLE_ENTITY;
+    error.meta = {dep, chain};
+    throw error;
+  }
+};
+
+const gemfileReducer = (lockFile, allDeps, ancestors) => (deps, dep) => {
+  const gemspec = lockFile.specs[dep];
+  // If for some reason a dependency isn't included in the specs then its
+  // better to just ignore it (otherwise all processing fails).
+  // This happens for bundler itself, it isn't included in the Gemfile.lock
+  // specs, even if its a dependency! (and that isn't documented anywhere)
+  if (gemspec) {
+    detectCycles(dep, ancestors);
+    if (allDeps.has(dep)) {
+      deps[dep] = allDeps.get(dep);
+    } else {
+      deps[dep] = {
+        name: dep,
+        version: gemspec.version,
+      };
+      allDeps.set(dep, deps[dep]);
+      deps[dep].dependencies = Object
+        .keys(gemspec)
+        .filter(k => k !== 'version')
+        .reduce(gemfileReducer(lockFile, allDeps, ancestors.concat([dep])), {});
+    }
+  }
+  return deps;
+};
+
+function gemfileLockToDependencies(fileContents) {
+  const lockFile = gemfile.interpret(fileContents, true);
+
+  return Object
+    .keys(lockFile.dependencies || {})
+    // this is required to sanitise git deps with no exact version
+    // listed as `rspec!`
+    .map(dep => dep.match(/[^!]+/)[0])
+    .reduce(gemfileReducer(lockFile, new Map(), []), {});
+}

src/lib/snyk-test/legacy.ts

@@ -0,0 +1,240 @@
+import * as _ from 'lodash';
+import * as depGraphLib from '@snyk/dep-graph';
+
+export {
+  convertTestDepGraphResultToLegacy,
+};
+
+interface Pkg {
+  name: string;
+  version?: string;
+}
+
+interface IssueData {
+  id: string;
+  packageName: string;
+  moduleName?: string;
+  semver: {
+    vulnerable: string | string[];
+    vulnerableHashes?: string[];
+    vulnerableByDistro?: {
+      [distroNameAndVersion: string]: string[];
+    }
+  };
+  patches: object[];
+  description: string;
+}
+
+interface AnnotatedIssue extends IssueData {
+  name: string;
+  version: string;
+  from: Array<string | boolean>;
+  upgradePath: Array<string | boolean>;
+  isUpgradable: boolean;
+  isPatchable: boolean;
+}
+
+interface LegacyVulnApiResult {
+  vulnerabilities: AnnotatedIssue[];
+  ok: boolean;
+  dependencyCount: number;
+  org: string;
+  policy: string;
+  isPrivate: boolean;
+  licensesPolicy: object | null;
+  packageManager: string;
+  ignoreSettings: object | null;
+  summary: string;
+  docker?: object;
+  severityThreshold?: string;
+}
+
+interface UpgradePathItem {
+  name: string;
+  version: string;
+  newVersion?: string;
+  isDropped?: boolean;
+}
+
+interface UpgradePath {
+  path: UpgradePathItem[];
+}
+
+interface FixInfo {
+  upgradePaths: UpgradePath[];
+  isPatchable: boolean;
+}
+
+interface TestDepGraphResult {
+  issuesData: {
+    [issueId: string]: IssueData;
+  };
+  affectedPkgs: {
+    [pkgId: string]: {
+      pkg: Pkg;
+      issues: {
+        [issueId: string]: {
+          issueId: string;
+          fixInfo: FixInfo;
+        };
+      };
+    };
+  };
+  docker: object;
+}
+
+interface TestDepGraphMeta {
+  isPublic: boolean;
+  isLicensesEnabled: boolean;
+  licensesPolicy?: {
+    severities: {
+      [type: string]: string;
+    };
+  };
+  ignoreSettings?: object;
+  policy: string;
+  org: string;
+}
+
+interface TestDeGraphResponse {
+  result: TestDepGraphResult;
+  meta: TestDepGraphMeta;
+}
+
+function convertTestDepGraphResultToLegacy(
+    res: TestDeGraphResponse,
+    depGraph: depGraphLib.DepGraph,
+    packageManager: string,
+    severityThreshold?: string): LegacyVulnApiResult {
+
+  const result = res.result;
+
+  const upgradePathsMap = new Map<string, string[]>();
+
+  for (const pkgInfo of _.values(result.affectedPkgs)) {
+    for (const pkgIssue of _.values(pkgInfo.issues)) {
+      if (pkgIssue.fixInfo && pkgIssue.fixInfo.upgradePaths) {
+        for (const upgradePath of pkgIssue.fixInfo.upgradePaths) {
+          const legacyFromPath = pkgPathToLegacyPath(upgradePath.path);
+          const vulnPathString = getVulnPathString(pkgIssue.issueId, legacyFromPath);
+          upgradePathsMap[vulnPathString] = toLegacyUpgradePath(upgradePath.path);
+        }
+      }
+    }
+  }
+
+  // generate the legacy vulns array (vuln-data + metada per vulnerable path).
+  //   use the upgradePathsMap to find available upgrade-paths
+  const vulns: AnnotatedIssue[] = [];
+
+  for (const pkgInfo of _.values(result.affectedPkgs)) {
+    for (const vulnPkgPath of depGraph.pkgPathsToRoot(pkgInfo.pkg)) {
+      const legacyFromPath = pkgPathToLegacyPath(vulnPkgPath.reverse());
+      for (const pkgIssue of _.values(pkgInfo.issues)) {
+        const vulnPathString = getVulnPathString(pkgIssue.issueId, legacyFromPath);
+        const upgradePath = upgradePathsMap[vulnPathString] || [];
+
+        // TODO: we need the full issue-data for every path only for the --json output,
+        //   consider picking only the required fields,
+        //   and append the full data only for --json, to minimize chance of out-of-memory
+        const annotatedIssue = Object.assign({}, result.issuesData[pkgIssue.issueId], {
+          from: legacyFromPath,
+          upgradePath,
+          isUpgradable: !!upgradePath[0] || !!upgradePath[1],
+          isPatchable: pkgIssue.fixInfo.isPatchable,
+          name: pkgInfo.pkg.name,
+          version: pkgInfo.pkg.version as string,
+        });
+
+        vulns.push(annotatedIssue);
+      }
+    }
+  }
+
+  const meta = res.meta || {};
+
+  severityThreshold = (severityThreshold === 'low') ? undefined : severityThreshold;
+
+  const legacyRes: LegacyVulnApiResult = {
+    vulnerabilities: vulns,
+    ok: vulns.length === 0,
+    dependencyCount: depGraph.getPkgs().length - 1,
+    org: meta.org,
+    policy: meta.policy,
+    isPrivate: !meta.isPublic,
+    licensesPolicy: meta.licensesPolicy || null,
+    packageManager,
+    ignoreSettings: meta.ignoreSettings || null,
+    docker: result.docker,
+    summary: getSummary(vulns, severityThreshold),
+    severityThreshold,
+  };
+
+  return legacyRes;
+}
+
+function getVulnPathString(issueId: string, vulnPath: string[]) {
+  return issueId + '|' +  JSON.stringify(vulnPath);
+}
+
+function pkgPathToLegacyPath(pkgPath: Pkg[]): string[] {
+  return pkgPath.map(toLegacyPkgId);
+}
+
+function toLegacyUpgradePath(upgradePath: UpgradePathItem[]): Array<string|boolean> {
+  return upgradePath
+    .filter((item) => !item.isDropped)
+    .map((item) => {
+      if (!item.newVersion) {
+        return false;
+      }
+
+      return `${item.name}@${item.newVersion}`;
+    });
+}
+
+function toLegacyPkgId(pkg: Pkg) {
+  return `${pkg.name}@${pkg.version || '*'}`;
+}
+
+function getSummary(vulns: object[], severityThreshold?: string): string {
+  const count = vulns.length;
+  let countText = '' + count;
+  const severityFilters: string[] = [];
+
+  const SEVERITIES = ['low', 'medium', 'high'];
+
+  if (severityThreshold) {
+    SEVERITIES.slice(SEVERITIES.indexOf(severityThreshold)).forEach((sev) => {
+      severityFilters.push(sev);
+    });
+  }
+
+  if (!count) {
+    if (severityFilters.length) {
+      return `No ${severityFilters.join(' or ')} severity vulnerabilities`;
+    }
+    return 'No known vulnerabilities';
+  }
+
+  if (severityFilters.length) {
+    countText += ' ' + severityFilters.join(' or ') + ' severity';
+  }
+
+  return `${countText} vulnerable dependency ${pl('path', count)}`;
+}
+
+function pl(word, count) {
+  const ext = {
+    y: 'ies',
+    default: 's',
+  };
+
+  const last = word.split('').pop();
+
+  if (count > 1) {
+    return word.slice(0, -1) + (ext[last] || last + ext.default);
+  }
+
+  return word;
+}

src/lib/snyk-test/npm/index.js

@@ -79,6 +79,7 @@ function test(root, options) {
           if (!pkg.name) {
             pkg.name = path.basename(path.resolve(root));
           }
+
           policyLocations = policyLocations.concat(pluckPolicies(pkg));
           debug('policies found', policyLocations);
           analytics.add('policies', policyLocations.length);