feat(core): add ability to use globs as outputs (nrwl#10894)

* feat(core): add ability to use globs as outputs

* fix(repo): adjust the way outputs are tracked

* docs(core): improve outputs documentation

docs/shared/configuration/packagejson.md

@@ -120,12 +120,49 @@ sources (non-test sources) of its dependencies. In other words, it treats test s
 
 ### outputs
 
-`"outputs": ["dist/libs/mylib"]` tells Nx where the `build` target is going to create file artifacts. The provided value
-is actually the default, so we can omit it in this case. `"outputs": []` tells Nx that the `test` target doesn't create
-any artifacts on disk.
+Targets may define outputs to tell Nx where the target is going to create file artifacts that Nx should cache. `"outputs": ["dist/libs/mylib"]` tells Nx where the `build` target is going to create file artifacts.
 
-This configuration is usually not needed. Nx comes with reasonable defaults (imported in `nx.json`) which implement the
-configuration above.
+This configuration is usually not needed. Nx comes with reasonable defaults (imported in `nx.json`) which implement the configuration above.
+
+#### Basic Example
+
+Usually, a target writes to a specific directory or a file. The following instructs Nx to cache `dist/libs/mylib` and `build/libs/mylib/main.js`:
+
+```json
+{
+  "build": {
+    "outputs": ["dist/libs/mylib", "build/libs/mylib/main.js"]
+  }
+}
+```
+
+#### Specifying Globs
+
+Sometimes, multiple targets might write to the same directory. When possible it is recommended to direct these targets into separate directories.
+
+```json
+{
+  "build-js": {
+    "outputs": ["dist/libs/mylib/js"]
+  },
+  "build-css": {
+    "outputs": ["dist/libs/mylib/css"]
+  }
+}
+```
+
+But if the above is not possible, globs can be specified as outputs to only cache a set of files rather than the whole directory.
+
+```json
+{
+  "build-js": {
+    "outputs": ["dist/libs/mylib/**/*.js"]
+  },
+  "build-css": {
+    "outputs": ["dist/libs/mylib/**/*.css"]
+  }
+}
+```
 
 ### dependsOn
 

docs/shared/configuration/projectjson.md

@@ -271,9 +271,85 @@ sources (non-test sources) of its dependencies. In other words, it treats test s
 
 ### Outputs
 
-`"outputs": ["dist/libs/mylib"]` tells Nx where the `build` target is going to create file artifacts. The provided value
-is actually the default, so we can omit it in this case. `"outputs": []` tells Nx that the `test` target doesn't create
-any artifacts on disk.
+Targets may define outputs to tell Nx where the target is going to create file artifacts that Nx should cache. `"outputs": ["dist/libs/mylib"]` tells Nx where the `build` target is going to create file artifacts.
+
+#### Basic Example
+
+Usually, a target writes to a specific directory or a file. The following instructs Nx to cache `dist/libs/mylib` and `build/libs/mylib/main.js`:
+
+```json
+  {
+    "build": {
+      ...,
+      "outputs": ["dist/libs/mylib", "build/libs/mylib/main.js"],
+      "options": {
+        ...
+      },
+    }
+  }
+```
+
+#### Referencing Options
+
+Most commonly, targets have an option for an output file or directory. Rather than duplicating the information as seen above, options can be referenced using the below syntax:
+
+> When the `outputPath` option is changed, Nx will start caching the new path as well.
+
+```json
+{
+  "build": {
+    ...,
+    "outputs": ["{options.outputPath}"],
+    "options": {
+      "outputPath": "dist/libs/mylib"
+    }
+  }
+}
+```
+
+#### Specifying Globs
+
+Sometimes, multiple targets might write to the same directory. When possible it is recommended to direct these targets into separate directories.
+
+```json
+{
+  "build-js": {
+    ...,
+    "outputs": ["dist/libs/mylib/js"],
+    "options": {
+      "outputPath": "dist/libs/mylib/js"
+    }
+  },
+  "build-css": {
+    ...,
+    "outputs": ["dist/libs/mylib/css"],
+    "options": {
+      "outputPath": "dist/libs/mylib/css"
+    }
+  }
+}
+```
+
+But if the above is not possible, globs can be specified as outputs to only cache a set of files rather than the whole directory.
+
+```json
+{
+  "build-js": {
+    ...,
+    "outputs": ["dist/libs/mylib/**/*.js"],
+    "options": {
+      "outputPath": "dist/libs/mylib"
+    }
+  },
+  "build-css": {
+    ...,
+    "outputs": ["dist/libs/mylib/**/*.css"],
+    "options": {
+      "outputPath": "dist/libs/mylib"
+    }
+  }
+}
+```
 
 ### dependsOn
 

e2e/nx-run/src/cache.test.ts

@@ -99,7 +99,7 @@ describe('cache', () => {
     expect(outputWithBothLintTasksCached).toContain(
       'read the output from the cache'
     );
-    expectCached(outputWithBothLintTasksCached, [
+    expectMatchedOutput(outputWithBothLintTasksCached, [
       myapp1,
       myapp2,
       `${myapp1}-e2e`,
@@ -164,6 +164,80 @@ describe('cache', () => {
     updateFile('nx.json', (c) => originalNxJson);
   }, 120000);
 
+  it('should support using globs as outputs', async () => {
+    const mylib = uniq('mylib');
+    runCLI(`generate @nrwl/workspace:library ${mylib}`);
+    updateProjectConfig(mylib, (c) => {
+      c.targets.build = {
+        executor: 'nx:run-commands',
+        outputs: ['dist/*.txt'],
+        options: {
+          commands: [
+            'rm -rf dist',
+            'mkdir dist',
+            'echo a > dist/a.txt',
+            'echo b > dist/b.txt',
+            'echo c > dist/c.txt',
+            'echo d > dist/d.txt',
+            'echo e > dist/e.txt',
+            'echo f > dist/f.txt',
+          ],
+          parallel: false,
+        },
+      };
+      return c;
+    });
+
+    // Run without cache
+    const runWithoutCache = runCLI(`build ${mylib}`);
+    expect(runWithoutCache).not.toContain('read the output from the cache');
+
+    // Rerun without touching anything
+    const rerunWithUntouchedOutputs = runCLI(`build ${mylib}`);
+    expect(rerunWithUntouchedOutputs).toContain(
+      'existing outputs match the cache'
+    );
+    const outputsWithUntouchedOutputs = listFiles('dist');
+    expect(outputsWithUntouchedOutputs).toContain('a.txt');
+    expect(outputsWithUntouchedOutputs).toContain('b.txt');
+    expect(outputsWithUntouchedOutputs).toContain('c.txt');
+    expect(outputsWithUntouchedOutputs).toContain('d.txt');
+    expect(outputsWithUntouchedOutputs).toContain('e.txt');
+    expect(outputsWithUntouchedOutputs).toContain('f.txt');
+
+    // Create a file in the dist that does not match output glob
+    updateFile('dist/c.ts', '');
+
+    // Rerun
+    const rerunWithNewUnrelatedFile = runCLI(`build ${mylib}`);
+    expect(rerunWithNewUnrelatedFile).toContain(
+      'existing outputs match the cache'
+    );
+    const outputsAfterAddingUntouchedFileAndRerunning = listFiles('dist');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('a.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('b.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('c.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('d.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('e.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('f.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('c.ts');
+
+    // Clear Dist
+    rmDist();
+
+    // Rerun
+    const rerunWithoutOutputs = runCLI(`build ${mylib}`);
+    expect(rerunWithoutOutputs).toContain('read the output from the cache');
+    const outputsWithoutOutputs = listFiles('dist');
+    expect(outputsWithoutOutputs).toContain('a.txt');
+    expect(outputsWithoutOutputs).toContain('b.txt');
+    expect(outputsWithoutOutputs).toContain('c.txt');
+    expect(outputsWithoutOutputs).toContain('d.txt');
+    expect(outputsWithoutOutputs).toContain('e.txt');
+    expect(outputsWithoutOutputs).toContain('f.txt');
+    expect(outputsWithoutOutputs).not.toContain('c.ts');
+  });
+
   it('should use consider filesets when hashing', async () => {
     const parent = uniq('parent');
     const child1 = uniq('child1');

packages/nx/src/tasks-runner/cache.spec.ts

@@ -0,0 +1,112 @@
+import { Cache, collapseExpandedOutputs } from './cache';
+
+describe('Cache', () => {
+  describe('collapseExpandedOutputs', () => {
+    it('should handle no outputs', async () => {
+      const outputs = [];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual([]);
+    });
+
+    it('should keep files as is', async () => {
+      const outputs = ['dist/apps/app1/0.js'];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['dist/apps/app1/0.js']);
+    });
+
+    it('should keep directories as is', async () => {
+      const outputs = ['dist/apps/app1'];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['dist/apps/app1']);
+    });
+
+    it('should keep short lists of directories as is', async () => {
+      const outputs = ['test-results/apps/app1', 'coverage/apps/app1'];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['test-results/apps/app1', 'coverage/apps/app1']);
+    });
+
+    it('should keep short lists of files as is', async () => {
+      const outputs = [
+        'test-results/apps/app1/results.xml',
+        'coverage/apps/app1/coverage.html',
+      ];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual([
+        'test-results/apps/app1/results.xml',
+        'coverage/apps/app1/coverage.html',
+      ]);
+    });
+
+    it('should collapse long lists of directories', async () => {
+      const outputs = [
+        'dist/apps/app1/a',
+        'dist/apps/app1/b',
+        'dist/apps/app1/c',
+        'dist/apps/app1/d',
+        'dist/apps/app1/e',
+        'dist/apps/app1/f',
+      ];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['dist/apps/app1']);
+    });
+
+    it('should collapse long lists of directories + files', async () => {
+      const outputs = [
+        'coverage/apps/app1',
+        'dist/apps/app1/a.txt',
+        'dist/apps/app1/b.txt',
+        'dist/apps/app1/c.txt',
+        'dist/apps/app1/d.txt',
+        'dist/apps/app1/e.txt',
+        'dist/apps/app1/f.txt',
+      ];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['coverage/apps/app1', 'dist/apps/app1']);
+    });
+
+    it('should keep long lists of top-level directories', async () => {
+      const outputs = ['a', 'b', 'c', 'd', 'e', 'f'];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['a', 'b', 'c', 'd', 'e', 'f']);
+    });
+
+    it('should collapse long lists of files', async () => {
+      const outputs = [
+        'dist/apps/app1/a.js',
+        'dist/apps/app1/b.js',
+        'dist/apps/app1/c.js',
+        'dist/apps/app1/d.js',
+        'dist/apps/app1/e.js',
+        'dist/apps/app1/f.js',
+      ];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['dist/apps/app1']);
+    });
+
+    it('should collapse long lists of files in nested directories', async () => {
+      const outputs = [];
+      // Create dist/apps/app1/n/m.js + dist/apps/app1/n/m.d.ts
+      for (let i = 0; i < 6; i++) {
+        outputs.push(`dist/apps/app1/${i}.js`);
+        outputs.push(`dist/apps/app1/${i}.d.ts`);
+        for (let j = 0; j < 6; j++) {
+          outputs.push(`dist/apps/app1/${i}/${j}.js`);
+          outputs.push(`dist/apps/app1/${i}/${j}.d.ts`);
+        }
+      }
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['dist/apps/app1']);
+    });
+  });
+});