Transitive Context Import Fix (#519)

* Add highlighting to directory trees

* Remove obsolete option from markdown-tree and get tsconfig to include vitest setup files

* Update docs on mocking

* Fix not being able to add extra options when declaring dirtree code fence in markdown

* Continue applying documentation fixes

* Update some documentation

* Apply lockfiles fix

* Fix plotly bundle incorrectly importing context

* Add an esbuild plugin to detect when context is being imported transitively from other bundles

* Change the context detection plugin to work for both bundles and tabs

* Update docs to talk about the need for isolating context dependent functions

* Make sure that package.json isn't read if manifest validation fails

* Properly handle root paths in convertPathsToPosix

* Fix incorrect paths referring to test mocks

* Update lint packages

* Update incorrect linting configuration

* Add paths configuration entry for shiki

* Update docs about module contexts

* Update devserver configuration to include bundles in hot reload

* Use Richard's implementation for lockfile change detection

* Use better memoization for exploring packages that have already been explored before

* Update docs

* Update broken tests and coverage inclusion

* Fix incorrect actions code

* Fix spelling

* fix: grammar

Author: leeyi45

.github/actions/src/__tests__/lockfiles.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it, vi } from 'vitest';
+import * as utils from '../lockfiles.js';
+
+vi.mock(import('../gitRoot.js'), () => ({
+  gitRoot: 'root'
+}));
+
+describe(utils.extractPackageName, () => {
+  it('works with packages that start with @', () => {
+    expect(utils.extractPackageName('@sourceacademy/tab-Rune@workspace:^'))
+      .toEqual('@sourceacademy/tab-Rune');
+  });
+
+  it('works with regular package names', () => {
+    expect(utils.extractPackageName('lodash@npm:^4.17.20'))
+      .toEqual('lodash');
+  });
+
+  it('throws an error on an invalid package name', () => {
+    expect(() => utils.extractPackageName('something weird'))
+      .toThrowError('Invalid package name: something weird');
+  });
+});

…s/src/lockfiles/__tests__/sample_why.txt …hub/actions/src/__tests__/sample_why.txt.github/actions/src/lockfiles/__tests__/sample_why.txt renamed to .github/actions/src/__tests__/sample_why.txt .github/actions/src/lockfiles/__tests__/sample_why.txt renamed to .github/actions/src/__tests__/sample_why.txt

@@ -4,7 +4,7 @@ import pathlib from 'path';
 import * as core from '@actions/core';
 import { describe, expect, test, vi } from 'vitest';
 import * as git from '../../commons.js';
-import * as lockfiles from '../../lockfiles/index.js';
+import * as lockfiles from '../../lockfiles.js';
 import { getAllPackages, getRawPackages, main } from '../index.js';
 
 const mockedCheckChanges = vi.spyOn(git, 'checkDirForChanges');

.github/actions/src/info/__tests__/index.test.ts

@@ -6,7 +6,7 @@ import type { SummaryTableRow } from '@actions/core/lib/summary.js';
 import packageJson from '../../../../package.json' with { type: 'json' };
 import { checkDirForChanges, type PackageRecord, type RawPackageRecord } from '../commons.js';
 import { gitRoot } from '../gitRoot.js';
-import { getPackagesWithResolutionChanges, hasLockFileChanged } from '../lockfiles/index.js';
+import { getPackagesWithResolutionChanges, hasLockFileChanged } from '../lockfiles.js';
 import { topoSortPackages } from './sorter.js';
 
 const packageNameRE = /^@sourceacademy\/(.+?)-(.+)$/u;
@@ -16,7 +16,7 @@ const packageNameRE = /^@sourceacademy\/(.+?)-(.+)$/u;
  * an unprocessed format
  */
 export async function getRawPackages(gitRoot: string, maxDepth?: number) {
-  let packagesWithResolutionChanges: Set<string> | null = null;
+  let packagesWithResolutionChanges: string[] | null = null;
 
   // If there are lock file changes we need to set hasChanges to true for
   // that package even if that package's directory has no changes
@@ -43,7 +43,7 @@ export async function getRawPackages(gitRoot: string, maxDepth?: number) {
 
             output[packageJson.name] = {
               directory: currentDir,
-              hasChanges: packagesWithResolutionChanges?.has(packageJson.name) ?? hasChanges,
+              hasChanges: hasChanges || !!packagesWithResolutionChanges?.includes(packageJson.name),
               package: packageJson
             };
           } catch (error) {

.github/actions/src/info/index.ts

@@ -0,0 +1,173 @@
+import fs from 'fs/promises';
+import pathlib from 'path';
+import * as core from '@actions/core';
+import { getExecOutput } from '@actions/exec';
+import memoize from 'lodash/memoize.js';
+import { extractPkgsFromYarnLockV2 } from 'snyk-nodejs-lockfile-parser';
+import { gitRoot } from './gitRoot.js';
+
+const packageNameRE = /^(.+)@.+$/;
+
+/**
+ * Lockfile specifications come in the form of package_name@resolution, but
+ * we only want the package name. This function extracts that package name,
+ * accounting for the fact that package names might start with '@'
+ */
+export function extractPackageName(raw: string) {
+  const match = packageNameRE.exec(raw);
+  if (!match) {
+    throw new Error(`Invalid package name: ${raw}`);
+  }
+
+  return match[1];
+}
+
+/**
+ * Parses and lockfile's contents and extracts all the different dependencies and
+ * versions
+ */
+function processLockFileText(contents: string) {
+  const lockFile = extractPkgsFromYarnLockV2(contents);
+  const mappings = new Set<string>();
+  for (const [pkgSpecifier, { resolution }] of Object.entries(lockFile)) {
+    mappings.add(`${pkgSpecifier} -> ${resolution}`);
+  }
+  return mappings;
+}
+
+/**
+ * Retrieves the contents of the lockfile in the repo
+ */
+async function getCurrentLockFile() {
+  const lockFilePath = pathlib.join(gitRoot, 'yarn.lock');
+  const contents = await fs.readFile(lockFilePath, 'utf-8');
+  return processLockFileText(contents);
+}
+
+/**
+ * Retrieves the contents of the lockfile on the master branch
+ */
+async function getMasterLockFile() {
+  const { stdout, stderr, exitCode } = await getExecOutput(
+    'git',
+    [
+      '--no-pager',
+      'show',
+      'origin/master:yarn.lock'
+    ],
+    { silent: true }
+  );
+
+  if (exitCode !== 0) {
+    core.error(stderr);
+    throw new Error('git show exited with non-zero error-code');
+  }
+
+  return processLockFileText(stdout);
+}
+
+interface ResolutionSpec { pkgSpecifier: string, pkgName: string }
+
+/**
+ * Parsed output entry returned by `yarn why`
+ */
+interface YarnWhyOutput {
+  value: string;
+  children: {
+    [locator: string]: {
+      locator: string;
+      descriptor: string;
+    };
+  };
+}
+
+/**
+ * Run `yarn why <package_name>` to see why a package is included
+ * Don't use recursive (-R) since we want to build the graph ourselves
+ * @function
+ */
+const runYarnWhy = memoize(async (pkgName: string) => {
+  // Memoize the call so that we don't need to call yarn why multiple times for each package
+  const { stdout: output, exitCode, stderr } = await getExecOutput('yarn', ['why', pkgName, '--json'], { silent: true });
+  if (exitCode !== 0) {
+    core.error(stderr);
+    throw new Error(`yarn why for ${pkgName} failed!`);
+  }
+
+  return output.split('\n').reduce<YarnWhyOutput[]>((res, each) => {
+    each = each.trim();
+    if (each === '') return res;
+
+    const pkg = JSON.parse(each) as YarnWhyOutput;
+    return [...res, pkg];
+  }, []);
+});
+
+/**
+ * Determines the names of the packages that have changed versions
+ */
+export async function getPackagesWithResolutionChanges() {
+  const [currentLockFileMappings, masterLockFileMappings] = await Promise.all([
+    getCurrentLockFile(),
+    getMasterLockFile()
+  ]);
+
+  const changes = new Set(masterLockFileMappings);
+  for (const edge of currentLockFileMappings) {
+    changes.delete(edge);
+  }
+
+  const frontier: ResolutionSpec[] = [];
+  const changedDeps = new Set<string>();
+  for (const edge of changes) {
+    const [pkgSpecifier] = edge.split(' -> ');
+    changedDeps.add(pkgSpecifier);
+    frontier.push({
+      pkgSpecifier,
+      pkgName: extractPackageName(pkgSpecifier)
+    });
+  }
+
+  while (frontier.length > 0) {
+    const { pkgName, pkgSpecifier } = frontier.shift()!;
+
+    const reasons = await runYarnWhy(pkgName);
+    reasons.forEach(pkg => {
+      if (changedDeps.has(pkg.value)) {
+        // If we've already added this pkg specifier, don't need to explore it again
+        return;
+      }
+
+      const childrenSpecifiers = Object.values(pkg.children).map(({ descriptor }) => descriptor);
+      if (!childrenSpecifiers.includes(pkgSpecifier)) return;
+
+      frontier.push({ pkgSpecifier: pkg.value, pkgName: extractPackageName(pkg.value) });
+      changedDeps.add(pkg.value);
+    });
+  }
+
+  core.info('=== Summary of dirty monorepo packages ===\n');
+  const pkgsToRebuild = [...changedDeps].filter(pkgSpecifier => pkgSpecifier.includes('@workspace:'));
+  for (const pkgName of pkgsToRebuild) {
+    core.info(`- ${pkgName}`);
+  }
+
+  return pkgsToRebuild.map(extractPackageName);
+}
+
+/**
+ * Returns `true` if there are changes present in the given directory relative to
+ * the master branch\
+ * Used to determine if the lockfile has changed
+ */
+export const hasLockFileChanged = memoize(async () => {
+  const { exitCode } = await getExecOutput(
+    'git --no-pager diff --quiet origin/master -- yarn.lock',
+    [],
+    {
+      failOnStdErr: false,
+      ignoreReturnCode: true
+    }
+  );
+  return exitCode !== 0;
+});