Dependency Graph Pipeline (Azure#7281)

abdulrahman305 · Feb 8, 2020 · 751aa55 · 751aa55
1 parent 3990f84
commit 751aa55
Show file tree

Hide file tree

Showing 3 changed files with 117 additions and 66 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Azure SDK for JavaScript
 
-[![Packages](https://img.shields.io/badge/packages-latest-blue.svg)](https://azure.github.io/azure-sdk/releases/latest/js.html) [![Dependencies](https://img.shields.io/badge/dependencies-analyzed-blue.svg)](https://azuresdkartifacts.blob.core.windows.net/azure-sdk-for-js/dependencies/dependencies.html)
+[![Packages](https://img.shields.io/badge/packages-latest-blue.svg)](https://azure.github.io/azure-sdk/releases/latest/js.html) [![Dependencies](https://img.shields.io/badge/dependencies-analyzed-blue.svg)](https://azuresdkartifacts.blob.core.windows.net/azure-sdk-for-js/dependencies/dependencies.html)[![DependencyGraph](https://img.shields.io/badge/dependencies-graph-blue.svg)](https://azuresdkartifacts.blob.core.windows.net/azure-sdk-for-js/dependencies/InterdependencyGraph.html)
 
 This repository is for active development of the Azure SDK for JavaScript (NodeJS & Browser). For consumers of the SDK we recommend visiting our [public developer docs](https://docs.microsoft.com/en-us/javascript/azure/) or our versioned [developer docs](https://azure.github.io/azure-sdk-for-js).
 

diff --git a/eng/pipelines/templates/jobs/aggregate-dependencies.yml b/eng/pipelines/templates/jobs/aggregate-dependencies.yml
@@ -0,0 +1,39 @@
+jobs:
+- job: 'ValidateDependencies'
+
+  pool:
+    vmImage: 'windows-2019'
+
+  steps:
+  - task: NodeTool@0
+    displayName: 'Use Node 10.x'
+    inputs:
+      versionSpec: 10.x
+
+  - script: 'npm ci'
+    workingDirectory: '$(Build.SourcesDirectory)/eng/tools/analyze-deps'
+    displayName: 'Install tool dependencies'
+
+  - script: |
+      node index.js --verbose --out "$(Build.ArtifactStagingDirectory)/dependencies.html" --dump "$(Build.ArtifactStagingDirectory)/data.js"
+    workingDirectory: '$(Build.SourcesDirectory)/eng/tools/analyze-deps'
+    displayName: 'Validate dependencies'
+
+  - pwsh: |
+      copy eng/common/InterdependencyGraph.html $(Build.ArtifactStagingDirectory)
+    displayName: 'Copy static file'
+
+  - task: AzureFileCopy@3
+    displayName: 'Upload dependency report'
+    inputs:
+      sourcePath: '$(Build.ArtifactStagingDirectory)'
+      additionalArgumentsForBlobCopy: |
+        '/Y'
+        '/Pattern:*'
+        '/S'
+        '/V'
+      azureSubscription: 'Azure SDK Artifacts'
+      destination: AzureBlob
+      storage: azuresdkartifacts
+      containerName: 'azure-sdk-for-js'
+      blobPrefix: dependencies
diff --git a/eng/tools/analyze-deps/index.js b/eng/tools/analyze-deps/index.js
@@ -208,8 +208,11 @@ const resolveRushPackageDeps = (packages, internalPackages, pnpmLock, pkgId, ext
     } else {
       // Local linked projects are not listed here, so pull the version from the local package.json
       const depInfo = Object.values(packages).find(pkgInfo => pkgInfo.name == dep.name);
-      dep.version = depInfo.version;
+      if (depInfo) {
+        dep.version = depInfo.version;
+      }
     }
+
   }
 };
 
@@ -226,85 +229,94 @@ const main = async () => {
     metavar: "DIR",
     help: "analyze packed tarballs in DIR rather than source packages in this repository"
   });
-  const args = parser.parseArgs();
+  try {
 
-  const context = {
-    packages: {},
-    dependencies: {},
-    external: [],
-    inconsistent: []
-  };
 
-  const rushPackages = await getRushPackages(path.resolve(`${__dirname}/../../../rush.json`));
-  context.packages = args.packdir
-    ? await getTarballPackages(path.resolve(args.packdir))
-    : rushPackages;
-  context.dependencies = constructDeps(context.packages);
-  context.external = Object.keys(context.dependencies).filter((p) => !(p in rushPackages));
-  context.inconsistent = Object.keys(context.dependencies).filter(
-    (p) => Object.keys(context.dependencies[p]).length > 1
-  );
+    const args = parser.parseArgs();
 
-  if (args.verbose) {
-    console.log("Packages analyzed:");
-    for (const package of Object.keys(context.packages).sort()) {
-      const info = context.packages[package];
-      console.log(`${package} ${info.ver}`);
-      console.log(`  from ${info.src}`);
-    }
+    const context = {
+      packages: {},
+      dependencies: {},
+      external: [],
+      inconsistent: []
+    };
 
-    console.log("\nDependencies discovered:");
-    for (const dep of Object.keys(context.dependencies).sort()) {
-      const info = context.dependencies[dep];
-      console.log(`${dep}`);
-      for (const ver of Object.keys(info).sort()) {
-        const pkgs = info[ver];
-        console.log(`${ver}`);
-        for (const pkg of pkgs.sort()) {
-          console.log(`  * ${pkg[0]} (${pkg[1]})`);
-        }
+    const rushPackages = await getRushPackages(path.resolve(`${__dirname}/../../../rush.json`));
+    context.packages = args.packdir
+      ? await getTarballPackages(path.resolve(args.packdir))
+      : rushPackages;
+    context.dependencies = constructDeps(context.packages);
+    context.external = Object.keys(context.dependencies).filter((p) => !(p in rushPackages));
+    context.inconsistent = Object.keys(context.dependencies).filter(
+      (p) => Object.keys(context.dependencies[p]).length > 1
+    );
+
+    if (args.verbose) {
+      console.log("Packages analyzed:");
+      for (const package of Object.keys(context.packages).sort()) {
+        const info = context.packages[package];
+        console.log(`${package} ${info.ver}`);
+        console.log(`  from ${info.src}`);
       }
-      console.log("");
-    }
 
-    for (const inc of context.inconsistent) {
-      const info = context.dependencies[inc];
-      const vers = Object.keys(info).sort();
-      console.log(`\nDependency '${inc}' has ${vers.length} unique specifiers:`);
-      for (const ver of vers.sort()) {
-        const pkgs = info[ver];
-        console.log(`'${ver}'`);
-        console.log(`${"-".repeat(ver.length + 2)}`);
-        for (const pkg of pkgs.sort()) {
-          console.log(`  * ${pkg[0]} (${pkg[1]})`);
+      console.log("\nDependencies discovered:");
+      for (const dep of Object.keys(context.dependencies).sort()) {
+        const info = context.dependencies[dep];
+        console.log(`${dep}`);
+        for (const ver of Object.keys(info).sort()) {
+          const pkgs = info[ver];
+          console.log(`${ver}`);
+          for (const pkg of pkgs.sort()) {
+            console.log(`  * ${pkg[0]} (${pkg[1]})`);
+          }
         }
         console.log("");
       }
+
+      for (const inc of context.inconsistent) {
+        const info = context.dependencies[inc];
+        const vers = Object.keys(info).sort();
+        console.log(`\nDependency '${inc}' has ${vers.length} unique specifiers:`);
+        for (const ver of vers.sort()) {
+          const pkgs = info[ver];
+          console.log(`'${ver}'`);
+          console.log(`${"-".repeat(ver.length + 2)}`);
+          for (const pkg of pkgs.sort()) {
+            console.log(`  * ${pkg[0]} (${pkg[1]})`);
+          }
+          console.log("");
+        }
+      }
     }
-  }
 
-  if (context.inconsistent.length > 0) {
-    if (!args.verbose) {
-      console.log(
-        "Incompatible dependency versions detected in libraries, run this script with --verbose for details"
-      );
+    if (context.inconsistent.length > 0) {
+      if (!args.verbose) {
+        console.log(
+          "Incompatible dependency versions detected in libraries, run this script with --verbose for details"
+        );
+      }
+    } else {
+      console.log("All library dependencies verified, no incompatible versions detected");
     }
-  } else {
-    console.log("All library dependencies verified, no incompatible versions detected");
-  }
 
-  if (args.out) {
-    await render(context, args.out);
-  }
+    if (args.out) {
+      await render(context, args.out);
+    }
 
-  if (args.dump) {
-    const internalPackages = Object.keys(rushPackages);
-    const dumpData = dumpRushPackages(context.packages);
-    const pnpmLock = await readPnpmLock(path.resolve(`${__dirname}/../../../common/config/rush/pnpm-lock.yaml`));
-    for (const pkgId of Object.keys(dumpData)) {
-      resolveRushPackageDeps(dumpData, internalPackages, pnpmLock, pkgId, args.external);
+    if (args.dump) {
+      const internalPackages = Object.keys(rushPackages);
+      const dumpData = dumpRushPackages(context.packages);
+      const pnpmLock = await readPnpmLock(path.resolve(`${__dirname}/../../../common/config/rush/pnpm-lock.yaml`));
+      for (const pkgId of Object.keys(dumpData)) {
+        resolveRushPackageDeps(dumpData, internalPackages, pnpmLock, pkgId, args.external);
+      }
+      await writeFile(args.dump, "const data = " + JSON.stringify(dumpData) + ";");
     }
-    await writeFile(args.dump, "const data = " + JSON.stringify(dumpData) + ";");
+  }
+  catch (ex) {
+    console.error(ex);
+  }
+  finally {
   }
 };