Merge pull request #4750 from aramissennyeydd/clustering-fixes

[rush]: fix operation clustering for shards and no-ops

Author: octogonz

common/changes/@microsoft/rush/clustering-fixes_2024-05-29-16-51.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Fixes build cache no-op and sharded operation clustering.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/reviews/api/rush-lib.api.md

@@ -1398,6 +1398,12 @@ export class RushLifecycleHooks {
 export class RushProjectConfiguration {
     readonly disableBuildCacheForProject: boolean;
     getCacheDisabledReason(trackedFileNames: Iterable<string>, phaseName: string, isNoOp: boolean): string | undefined;
+    static getCacheDisabledReasonForProject(options: {
+        projectConfiguration: RushProjectConfiguration | undefined;
+        trackedFileNames: Iterable<string>;
+        phaseName: string;
+        isNoOp: boolean;
+    }): string | undefined;
     readonly incrementalBuildIgnoredGlobs: ReadonlyArray<string>;
     // (undocumented)
     readonly operationSettingsByOperationName: ReadonlyMap<string, Readonly<IOperationSettings>>;

libraries/rush-lib/src/api/RushProjectConfiguration.ts

@@ -391,6 +391,32 @@ export class RushProjectConfiguration {
     }
   }
 
+  /**
+   * Source of truth for whether a project is unable to use the build cache for a given phase.
+   * As some operations may not have a rush-project.json file defined at all, but may be no-op operations
+   *  we'll want to ignore those completely.
+   */
+  public static getCacheDisabledReasonForProject(options: {
+    projectConfiguration: RushProjectConfiguration | undefined;
+    trackedFileNames: Iterable<string>;
+    phaseName: string;
+    isNoOp: boolean;
+  }): string | undefined {
+    const { projectConfiguration, trackedFileNames, phaseName, isNoOp } = options;
+    if (isNoOp) {
+      return undefined;
+    }
+
+    if (!projectConfiguration) {
+      return (
+        `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` +
+        'or one provided by a rig, so it does not support caching.'
+      );
+    }
+
+    return projectConfiguration.getCacheDisabledReason(trackedFileNames, phaseName, isNoOp);
+  }
+
   /**
    * Loads the rush-project.json data for the specified project.
    */

libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts

@@ -11,8 +11,7 @@ import type { Operation } from './Operation';
 import { clusterOperations, type IOperationBuildCacheContext } from './CacheableOperationPlugin';
 import { DisjointSet } from '../cobuild/DisjointSet';
 import type { IOperationExecutionResult } from './IOperationExecutionResult';
-import { RushConstants } from '../RushConstants';
-import type { RushProjectConfiguration } from '../../api/RushProjectConfiguration';
+import { RushProjectConfiguration } from '../../api/RushProjectConfiguration';
 
 const PLUGIN_NAME: 'BuildPlanPlugin' = 'BuildPlanPlugin';
 
@@ -64,14 +63,16 @@ export class BuildPlanPlugin implements IPhasedCommandPlugin {
             projectConfigurations.get(associatedProject);
           const fileHashes: Map<string, string> | undefined =
             await projectChangeAnalyzer._tryGetProjectDependenciesAsync(associatedProject, terminal);
-          const cacheDisabledReason: string | undefined = projectConfiguration
-            ? projectConfiguration.getCacheDisabledReason(
-                fileHashes!.keys(),
-                associatedPhase.name,
-                operation.isNoOp
-              )
-            : `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` +
-              'or one provided by a rig, so it does not support caching.';
+          if (!fileHashes) {
+            continue;
+          }
+          const cacheDisabledReason: string | undefined =
+            RushProjectConfiguration.getCacheDisabledReasonForProject({
+              projectConfiguration,
+              trackedFileNames: fileHashes.keys(),
+              isNoOp: operation.isNoOp,
+              phaseName: associatedPhase.name
+            });
           buildCacheByOperation.set(operation, { cacheDisabledReason });
         }
       }

libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts

@@ -12,7 +12,7 @@ import { OperationStatus } from './OperationStatus';
 import { CobuildLock, type ICobuildCompletedState } from '../cobuild/CobuildLock';
 import { ProjectBuildCache } from '../buildCache/ProjectBuildCache';
 import { RushConstants } from '../RushConstants';
-import type { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration';
+import { type IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration';
 import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync';
 import { ProjectLogWritable } from './ProjectLogWritable';
 import type { CobuildConfiguration } from '../../api/CobuildConfiguration';
@@ -123,10 +123,13 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
               );
             }
 
-            const cacheDisabledReason: string | undefined = projectConfiguration
-              ? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName, operation.isNoOp)
-              : `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` +
-                'or one provided by a rig, so it does not support caching.';
+            const cacheDisabledReason: string | undefined =
+              RushProjectConfiguration.getCacheDisabledReasonForProject({
+                projectConfiguration,
+                phaseName: phaseName,
+                isNoOp: operation.isNoOp,
+                trackedFileNames: fileHashes.keys()
+              });
 
             disjointSet?.add(operation);