Watch extended configs if present (#41493)

* Watch extended configs if present

* Address code review comments

Added new `WatchType` for extended config files. Refactored watch map
update to separate function, relocated call sites. Removed unnecessary
test cases and relocated with new tests in programUpdates.

* Unify extended config file watching between tsc/tsserver

Update `updateExtendedConfigFilesWatch` to read from a
`TsConfigSourceFile` to get `extendedSourceFiles`. Add watcher map to
`ConfiguredProject` in the server. New test cases to verify correct
events triggered and extended files are being watched properly.

* Simplify watcher callback, fix tests

Removes unnecessary actions in extended config watcher callback
function. Updates tests to match.

* Share extended config watchers across projects in server

New shared watcher map in ProjectService that stores callbacks per
project to be invoked when the file watcher is triggered. The
FileWatcher is created with the watch options of the first Project to
watch the extended config.

* Refactor shared extended config map and watchers

Remove all server-related utility functions/types from
watchUtilities. Store config-project mapping and config file watchers
inside ProjectService with new private methods to add or remove
projects.

* Store projects in extended config file watcher

Creates SharedExtendedConfigFileWatcher in both editorServices
(tsserver) and tsbuildPublic. The file watcher is responsible for
triggering a full project reload for the contained projects. Upon
reload, any configs that are no longer related to a project have their
watchers updated to match. New test cases to confirm that the file
watchers for extended configs are closed when the project is closed.

* Apply suggestions from code review

Co-authored-by: Sheetal Nandi <shkamat@microsoft.com>

* Map extended config files by path

* Move shared watcher into utilities and add more tests

Co-authored-by: Sheetal Nandi <shkamat@microsoft.com>

Author: molisani

src/compiler/tsbuildPublic.ts

@@ -254,6 +254,7 @@ namespace ts {
         readonly allWatchedWildcardDirectories: ESMap<ResolvedConfigFilePath, ESMap<string, WildcardDirectoryWatcher>>;
         readonly allWatchedInputFiles: ESMap<ResolvedConfigFilePath, ESMap<Path, FileWatcher>>;
         readonly allWatchedConfigFiles: ESMap<ResolvedConfigFilePath, FileWatcher>;
+        readonly allWatchedExtendedConfigFiles: ESMap<Path, SharedExtendedConfigFileWatcher<ResolvedConfigFilePath>>;
 
         timerToBuildInvalidatedProject: any;
         reportFileChangeDetected: boolean;
@@ -325,6 +326,7 @@ namespace ts {
             allWatchedWildcardDirectories: new Map(),
             allWatchedInputFiles: new Map(),
             allWatchedConfigFiles: new Map(),
+            allWatchedExtendedConfigFiles: new Map(),
 
             timerToBuildInvalidatedProject: undefined,
             reportFileChangeDetected: false,
@@ -462,6 +464,15 @@ namespace ts {
                 { onDeleteValue: closeFileWatcher }
             );
 
+            state.allWatchedExtendedConfigFiles.forEach(watcher => {
+                watcher.projects.forEach(project => {
+                    if (!currentProjects.has(project)) {
+                        watcher.projects.delete(project);
+                    }
+                });
+                watcher.close();
+            });
+
             mutateMapSkippingNewValues(
                 state.allWatchedWildcardDirectories,
                 currentProjects,
@@ -1165,6 +1176,7 @@ namespace ts {
 
             if (reloadLevel === ConfigFileProgramReloadLevel.Full) {
                 watchConfigFile(state, project, projectPath, config);
+                watchExtendedConfigFiles(state, projectPath, config);
                 watchWildCardDirectories(state, project, projectPath, config);
                 watchInputFiles(state, project, projectPath, config);
             }
@@ -1789,6 +1801,24 @@ namespace ts {
         ));
     }
 
+    function watchExtendedConfigFiles(state: SolutionBuilderState, resolvedPath: ResolvedConfigFilePath, parsed: ParsedCommandLine | undefined) {
+        updateSharedExtendedConfigFileWatcher(
+            resolvedPath,
+            parsed,
+            state.allWatchedExtendedConfigFiles,
+            (extendedConfigFileName, extendedConfigFilePath) => state.watchFile(
+                extendedConfigFileName,
+                () => state.allWatchedExtendedConfigFiles.get(extendedConfigFilePath)?.projects.forEach(projectConfigFilePath =>
+                    invalidateProjectAndScheduleBuilds(state, projectConfigFilePath, ConfigFileProgramReloadLevel.Full)
+                ),
+                PollingInterval.High,
+                parsed?.watchOptions,
+                WatchType.ExtendedConfigFile,
+            ),
+            fileName => toPath(state, fileName),
+        );
+    }
+
     function watchWildCardDirectories(state: SolutionBuilderState, resolved: ResolvedConfigFileName, resolvedPath: ResolvedConfigFilePath, parsed: ParsedCommandLine) {
         if (!state.watch) return;
         updateWatchingWildcardDirectories(
@@ -1846,6 +1876,7 @@ namespace ts {
             const cfg = parseConfigFile(state, resolved, resolvedPath);
             // Watch this file
             watchConfigFile(state, resolved, resolvedPath, cfg);
+            watchExtendedConfigFiles(state, resolvedPath, cfg);
             if (cfg) {
                 // Update watchers for wildcard directories
                 watchWildCardDirectories(state, resolved, resolvedPath, cfg);
@@ -1858,6 +1889,10 @@ namespace ts {
 
     function stopWatching(state: SolutionBuilderState) {
         clearMap(state.allWatchedConfigFiles, closeFileWatcher);
+        clearMap(state.allWatchedExtendedConfigFiles, watcher => {
+            watcher.projects.clear();
+            watcher.close();
+        });
         clearMap(state.allWatchedWildcardDirectories, watchedWildcardDirectories => clearMap(watchedWildcardDirectories, closeFileWatcherOf));
         clearMap(state.allWatchedInputFiles, watchedWildcardDirectories => clearMap(watchedWildcardDirectories, closeFileWatcher));
     }

src/compiler/watch.ts

@@ -409,6 +409,7 @@ namespace ts {
     export type WatchType = WatchTypeRegistry[keyof WatchTypeRegistry];
     export const WatchType: WatchTypeRegistry = {
         ConfigFile: "Config file",
+        ExtendedConfigFile: "Extended config file",
         SourceFile: "Source file",
         MissingFile: "Missing file",
         WildcardDirectory: "Wild card directory",
@@ -418,6 +419,7 @@ namespace ts {
 
     export interface WatchTypeRegistry {
         ConfigFile: "Config file",
+        ExtendedConfigFile: "Extended config file",
         SourceFile: "Source file",
         MissingFile: "Missing file",
         WildcardDirectory: "Wild card directory",

src/compiler/watchPublic.ts

@@ -246,6 +246,7 @@ namespace ts {
 
         let builderProgram: T;
         let reloadLevel: ConfigFileProgramReloadLevel;                      // level to indicate if the program needs to be reloaded from config file/just filenames etc
+        let extendedConfigFilesMap: ESMap<Path, FileWatcher>;               // Map of file watchers for the extended config files
         let missingFilesMap: ESMap<Path, FileWatcher>;                        // Map of file watchers for the missing files
         let watchedWildcardDirectories: ESMap<string, WildcardDirectoryWatcher>; // map of watchers for the wild card directories in the config file
         let timerToUpdateProgram: any;                                      // timer callback to recompile the program
@@ -337,6 +338,9 @@ namespace ts {
         // Update the wild card directory watch
         watchConfigFileWildCardDirectories();
 
+        // Update extended config file watch
+        watchExtendedConfigFiles();
+
         return configFileName ?
             { getCurrentProgram: getCurrentBuilderProgram, getProgram: updateProgram, close } :
             { getCurrentProgram: getCurrentBuilderProgram, getProgram: updateProgram, updateRootFileNames, close };
@@ -354,6 +358,10 @@ namespace ts {
                 configFileWatcher.close();
                 configFileWatcher = undefined;
             }
+            if (extendedConfigFilesMap) {
+                clearMap(extendedConfigFilesMap, closeFileWatcher);
+                extendedConfigFilesMap = undefined!;
+            }
             if (watchedWildcardDirectories) {
                 clearMap(watchedWildcardDirectories, closeFileWatcherOf);
                 watchedWildcardDirectories = undefined!;
@@ -657,6 +665,9 @@ namespace ts {
 
             // Update the wild card directory watch
             watchConfigFileWildCardDirectories();
+
+            // Update extended config file watch
+            watchExtendedConfigFiles();
         }
 
         function parseConfigFile() {
@@ -777,5 +788,23 @@ namespace ts {
                 WatchType.WildcardDirectory
             );
         }
+
+        function watchExtendedConfigFiles() {
+            // Update the extended config files watcher
+            mutateMap(
+                extendedConfigFilesMap ||= new Map(),
+                arrayToMap(compilerOptions.configFile?.extendedSourceFiles || emptyArray, toPath),
+                {
+                    // Watch the extended config files
+                    createNewValue: watchExtendedConfigFile,
+                    // Config files that are no longer extended should no longer be watched.
+                    onDeleteValue: closeFileWatcher
+                }
+            );
+        }
+
+        function watchExtendedConfigFile(extendedConfigFile: Path) {
+            return watchFile(extendedConfigFile, scheduleProgramReload, PollingInterval.High, watchOptions, WatchType.ExtendedConfigFile);
+        }
     }
 }

src/compiler/watchUtilities.ts

@@ -257,6 +257,51 @@ namespace ts {
         Full
     }
 
+    export interface SharedExtendedConfigFileWatcher<T> extends FileWatcher {
+        fileWatcher: FileWatcher;
+        projects: Set<T>;
+    }
+
+    /**
+     * Updates the map of shared extended config file watches with a new set of extended config files from a base config file of the project
+     */
+    export function updateSharedExtendedConfigFileWatcher<T>(
+        projectPath: T,
+        parsed: ParsedCommandLine | undefined,
+        extendedConfigFilesMap: ESMap<Path, SharedExtendedConfigFileWatcher<T>>,
+        createExtendedConfigFileWatch: (extendedConfigPath: string, extendedConfigFilePath: Path) => FileWatcher,
+        toPath: (fileName: string) => Path,
+    ) {
+        const extendedConfigs = arrayToMap(parsed?.options.configFile?.extendedSourceFiles || emptyArray, toPath);
+        // remove project from all unrelated watchers
+        extendedConfigFilesMap.forEach((watcher, extendedConfigFilePath) => {
+            if (!extendedConfigs.has(extendedConfigFilePath)) {
+                watcher.projects.delete(projectPath);
+                watcher.close();
+            }
+        });
+        // Update the extended config files watcher
+        extendedConfigs.forEach((extendedConfigFileName, extendedConfigFilePath) => {
+            const existing = extendedConfigFilesMap.get(extendedConfigFilePath);
+            if (existing) {
+                existing.projects.add(projectPath);
+            }
+            else {
+                // start watching previously unseen extended config
+                extendedConfigFilesMap.set(extendedConfigFilePath, {
+                    projects: new Set([projectPath]),
+                    fileWatcher: createExtendedConfigFileWatch(extendedConfigFileName, extendedConfigFilePath),
+                    close: () => {
+                        const existing = extendedConfigFilesMap.get(extendedConfigFilePath);
+                        if (!existing || existing.projects.size !== 0) return;
+                        existing.fileWatcher.close();
+                        extendedConfigFilesMap.delete(extendedConfigFilePath);
+                    },
+                });
+            }
+        });
+    }
+
     /**
      * Updates the existing missing file watches with the new set of missing files after new program is created
      */

src/server/editorServices.ts

@@ -756,6 +756,9 @@ namespace ts.server {
         /*@internal*/
         readonly watchFactory: WatchFactory<WatchType, Project>;
 
+        /*@internal*/
+        private readonly sharedExtendedConfigFileWatchers = new Map<Path, SharedExtendedConfigFileWatcher<NormalizedPath>>();
+
         /*@internal*/
         readonly packageJsonCache: PackageJsonCache;
         /*@internal*/
@@ -1350,6 +1353,43 @@ namespace ts.server {
             }
         }
 
+        /*@internal*/
+        updateSharedExtendedConfigFileMap({ canonicalConfigFilePath }: ConfiguredProject, parsedCommandLine: ParsedCommandLine) {
+            updateSharedExtendedConfigFileWatcher(
+                canonicalConfigFilePath,
+                parsedCommandLine,
+                this.sharedExtendedConfigFileWatchers,
+                (extendedConfigFileName, extendedConfigFilePath) => this.watchFactory.watchFile(
+                    extendedConfigFileName,
+                    () => {
+                        let ensureProjectsForOpenFiles = false;
+                        this.sharedExtendedConfigFileWatchers.get(extendedConfigFilePath)?.projects.forEach(canonicalPath => {
+                            const project = this.configuredProjects.get(canonicalPath);
+                            // Skip refresh if project is not yet loaded
+                            if (!project || project.isInitialLoadPending()) return;
+                            project.pendingReload = ConfigFileProgramReloadLevel.Full;
+                            project.pendingReloadReason = `Change in extended config file ${extendedConfigFileName} detected`;
+                            this.delayUpdateProjectGraph(project);
+                            ensureProjectsForOpenFiles = true;
+                        });
+                        if (ensureProjectsForOpenFiles) this.delayEnsureProjectForOpenFiles();
+                    },
+                    PollingInterval.High,
+                    this.hostConfiguration.watchOptions,
+                    WatchType.ExtendedConfigFile
+                ),
+                fileName => this.toPath(fileName),
+            );
+        }
+
+        /*@internal*/
+        removeProjectFromSharedExtendedConfigFileMap(project: ConfiguredProject) {
+            this.sharedExtendedConfigFileWatchers.forEach(watcher => {
+                watcher.projects.delete(project.canonicalConfigFilePath);
+                watcher.close();
+            });
+        }
+
         /**
          * This is the callback function for the config file add/remove/change at any location
          * that matters to open script info but doesnt have configured project open
@@ -2051,7 +2091,6 @@ namespace ts.server {
                 this,
                 this.documentRegistry,
                 cachedDirectoryStructureHost);
-            // TODO: We probably should also watch the configFiles that are extended
             project.createConfigFileWatcher();
             this.configuredProjects.set(project.canonicalConfigFilePath, project);
             this.setConfigFileExistenceByNewConfiguredProject(project);
@@ -2134,12 +2173,14 @@ namespace ts.server {
             if (lastFileExceededProgramSize) {
                 project.disableLanguageService(lastFileExceededProgramSize);
                 project.stopWatchingWildCards();
+                this.removeProjectFromSharedExtendedConfigFileMap(project);
             }
             else {
                 project.setCompilerOptions(compilerOptions);
                 project.setWatchOptions(parsedCommandLine.watchOptions);
                 project.enableLanguageService();
                 project.watchWildcards(new Map(getEntries(parsedCommandLine.wildcardDirectories!))); // TODO: GH#18217
+                this.updateSharedExtendedConfigFileMap(project, parsedCommandLine);
             }
             project.enablePluginsWithOptions(compilerOptions, this.currentPluginConfigOverrides);
             const filesToAdd = parsedCommandLine.fileNames.concat(project.getExternalFiles());

src/server/project.ts

@@ -2292,6 +2292,7 @@ namespace ts.server {
             }
 
             this.stopWatchingWildCards();
+            this.projectService.removeProjectFromSharedExtendedConfigFileMap(this);
             this.projectErrors = undefined;
             this.openFileWatchTriggered.clear();
             this.compilerHost = undefined;