WI #2026 #2027 Avoid FileCompiler leaks in LanguageServer (#2030)

fm-117 · web-flow · commit 94208b4c5077 · 2021-09-06T10:12:42.000+02:00
* WI #2027 Avoid FileCompiler leaks in Workspace * WI #2027 Fix LSR test * WI #2026 Remove static END_OF_FILE token (#2031)
diff --git a/TypeCobol.LanguageServer.Test/LSRTests/ProcedureCompletionNoParam/input/ProcedureCompletionNoParam.tlsp b/TypeCobol.LanguageServer.Test/LSRTests/ProcedureCompletionNoParam/input/ProcedureCompletionNoParam.tlsp
@@ -98,10 +98,6 @@
       "category": 0,
       "message": "{\"jsonrpc\":\"2.0\",\"method\":\"textDocument/didSave\",\"params\":{\"textDocument\":{\"version\":0,\"uri\":\"file:/C:/Users/LANOYDO/AppData/Local/Temp/1/tcbl/DVZZDLY1103830253336368444.cee\"},\"text\":\"       IDENTIFICATION DIVISION.\\r\\n       PROGRAM-ID. DVZZBCO0.\\r\\n       DATA DIVISION.\\r\\n       WORKING-STORAGE SECTION.\\r\\n       PROCEDURE DIVISION.\\r\\n           call DVZZSTKD::Foo\\r\\n           .\\r\\n       END PROGRAM DVZZBCO0.\\r\\n\\r\\n       IDENTIFICATION DIVISION.\\r\\n       PROGRAM-ID. DVZZSTKD.\\r\\n       PROCEDURE DIVISION.\\r\\n       declare procedure Foo public.\\r\\n       procedure division.\\r\\n           goback.\\r\\n       end-declare.\\r\\n\\r\\n       END PROGRAM DVZZSTKD.\"}}"
     },
-    {
-      "category": 1,
-      "message": "{\"jsonrpc\":\"2.0\",\"method\":\"textDocument/publishDiagnostics\",\"params\":{\"uri\":\"file:///C:/Users/LANOYDO/AppData/Local/Temp/1/tcbl/DVZZDLY1103830253336368444.cee\",\"diagnostics\":[]}}"
-    },
     {
       "category": 1,
       "message": "{\"jsonrpc\":\"2.0\",\"method\":\"textDocument/publishDiagnostics\",\"params\":{\"uri\":\"file:///C:/Users/LANOYDO/AppData/Local/Temp/1/tcbl/DVZZDLY1103830253336368444.cee\",\"diagnostics\":[]}}"
diff --git a/TypeCobol.LanguageServer/Workspace.cs b/TypeCobol.LanguageServer/Workspace.cs
@@ -45,7 +45,7 @@ public class Workspace
 
         internal CompilationProject CompilationProject { get; private set; }
 
-        private List<FileCompiler> _fileCompilerWaittingForNodePhase;
+        private List<FileCompiler> _fileCompilerWaitingForNodePhase;
         public TypeCobolConfiguration Configuration { get; private set; }
         public event EventHandler<DiagnosticEvent> DiagnosticsEvent;
         public event EventHandler<EventArgs> DocumentModifiedEvent;
@@ -162,7 +162,7 @@ public Workspace(string rootDirectoryFullName, string workspaceName, Queue<Messa
             MessagesActionsQueue = messagesActionsQueue;
             Configuration = new TypeCobolConfiguration();
             _openedDocuments = new Dictionary<Uri, DocumentContext>();
-            _fileCompilerWaittingForNodePhase = new List<FileCompiler>();
+            _fileCompilerWaitingForNodePhase = new List<FileCompiler>();
             _Logger = logger;
 
             this._rootDirectoryFullName = rootDirectoryFullName;
@@ -234,22 +234,22 @@ private FileCompiler OpenTextDocument(DocumentContext docContext, string sourceT
 #else
             fileCompiler = new FileCompiler(initialTextDocumentLines, CompilationProject.SourceFileProvider, CompilationProject, CompilationProject.CompilationOptions, _customSymbols, CompilationProject);
 #endif
-            //Set Any Language Server Connection Options.
-            docContext.FileCompiler = fileCompiler;
-            docContext.LanguageServerConnection(true);
-
-            fileCompiler.CompilationResultsForProgram.UpdateTokensLines();
 
             lock (_lockForOpenedDocuments)
             {
                 if (_openedDocuments.ContainsKey(docContext.Uri))
                     CloseSourceFile(docContext.Uri); //Close and remove the previous opened file.
 
                 _openedDocuments.Add(docContext.Uri, docContext);
-                fileCompiler.CompilationResultsForProgram.CodeAnalysisCompleted += FinalCompilationStepCompleted;
             }
 
+            //Set Any Language Server Connection Options.
+            docContext.FileCompiler = fileCompiler;
+            docContext.LanguageServerConnection(true);
+
             fileCompiler.CompilationResultsForProgram.SetOwnerThread(Thread.CurrentThread);
+            fileCompiler.CompilationResultsForProgram.CodeAnalysisCompleted += FinalCompilationStepCompleted;
+            fileCompiler.CompilationResultsForProgram.UpdateTokensLines();
 
             if (lsrOptions != LsrTestingOptions.LsrSourceDocumentTesting)
             {
@@ -325,14 +325,14 @@ public void UpdateSourceFile(Uri fileUri, TextChangedEvent textChangedEvent)
                 {
                     if (!_timerDisabled) //If TimerDisabled is false, create a timer to automatically launch Node phase
                     {
-                        lock (_fileCompilerWaittingForNodePhase)
+                        lock (_fileCompilerWaitingForNodePhase)
                         {
-                            if (!_fileCompilerWaittingForNodePhase.Contains(fileCompilerToUpdate))
-                                _fileCompilerWaittingForNodePhase.Add(fileCompilerToUpdate); //Store that this fileCompiler will soon need a Node Phase
+                            if (!_fileCompilerWaitingForNodePhase.Contains(fileCompilerToUpdate))
+                                _fileCompilerWaitingForNodePhase.Add(fileCompilerToUpdate); //Store that this fileCompiler will soon need a Node Phase
                         }
 
                         _semanticUpdaterTimer = new System.Timers.Timer(750);
-                        _semanticUpdaterTimer.Elapsed += (sender, e) => TimerEvent(sender, e, fileCompilerToUpdate);
+                        _semanticUpdaterTimer.Elapsed += (sender, e) => TimerEvent(fileUri);
                         _semanticUpdaterTimer.Start();
                     }
                 }
@@ -392,23 +392,29 @@ private void ExecutionStepEventHandler(object oFileCompiler, ExecutionStepEventA
         /// <param name="sender"></param>
         /// <param name="eventArgs"></param>
         /// <param name="fileCompiler"></param>
-        private void TimerEvent(object sender, ElapsedEventArgs eventArgs, FileCompiler fileCompiler)
+        private void TimerEvent(Uri fileUri)
         {
             try
             {
                 _semanticUpdaterTimer.Stop();
-                Action nodeRefreshAction = () => { RefreshSyntaxTree(fileCompiler, SyntaxTreeRefreshLevel.RebuildNodesAndPerformQualityCheck); };
                 lock (MessagesActionsQueue)
                 {
-                    MessagesActionsQueue.Enqueue(new MessageActionWrapper(nodeRefreshAction));
+                    MessagesActionsQueue.Enqueue(new MessageActionWrapper(Refresh));
                 }
             }
             catch (Exception e)
             {
                 //In case Timer Thread crash
                 ExceptionTriggered(null, new ThreadExceptionEventArgs(e));
             }
-           
+
+            void Refresh()
+            {
+                if (TryGetOpenedDocumentContext(fileUri, out var docContext))
+                {
+                    RefreshSyntaxTree(docContext.FileCompiler, SyntaxTreeRefreshLevel.RebuildNodesAndPerformQualityCheck);
+                }
+            }
         }
 
         /// <summary>
@@ -446,12 +452,12 @@ public void RefreshSyntaxTree(FileCompiler fileCompiler, SyntaxTreeRefreshLevel
         {
             if (refreshLevel == SyntaxTreeRefreshLevel.NoRefresh) return; //nothing to do
 
-            lock (_fileCompilerWaittingForNodePhase)
+            lock (_fileCompilerWaitingForNodePhase)
             {
-                var fileCompilerNeedsRefresh = _fileCompilerWaittingForNodePhase.Contains(fileCompiler);
+                var fileCompilerNeedsRefresh = _fileCompilerWaitingForNodePhase.Contains(fileCompiler);
                 if (fileCompilerNeedsRefresh)
                 {
-                    _fileCompilerWaittingForNodePhase.Remove(fileCompiler);
+                    _fileCompilerWaitingForNodePhase.Remove(fileCompiler);
                 }
                 else
                 {
@@ -495,16 +501,27 @@ void RefreshCodeAnalysisResults()
         /// </summary>
         public void CloseSourceFile(Uri fileUri)
         {
+            FileCompiler fileCompilerToClose = null;
+
+            //Remove from opened documents dictionary
             lock (_lockForOpenedDocuments)
             {
-                if (_openedDocuments.ContainsKey(fileUri))
+                if (_openedDocuments.TryGetValue(fileUri, out var contextToClose))
                 {
-                    var contextToClose = _openedDocuments[fileUri];
-                    FileCompiler fileCompilerToClose = contextToClose.FileCompiler;
+                    fileCompilerToClose = contextToClose.FileCompiler;
                     _openedDocuments.Remove(fileUri);
                     fileCompilerToClose.CompilationResultsForProgram.CodeAnalysisCompleted -= FinalCompilationStepCompleted;
                 }
-            }            
+            }
+
+            //Remove from pending semantic analysis list
+            if (fileCompilerToClose != null)
+            {
+                lock (_fileCompilerWaitingForNodePhase)
+                {
+                    _fileCompilerWaitingForNodePhase.Remove(fileCompilerToClose);
+                }
+            }
         }
 
         /// <summary>
diff --git a/TypeCobol.Test/Parser/Preprocessor/PreprocessorUtils.cs b/TypeCobol.Test/Parser/Preprocessor/PreprocessorUtils.cs
@@ -73,7 +73,7 @@ private static string ProcessTokensDocument(ProcessedTokensDocument processedDoc
             StringBuilder sbTokens = new StringBuilder();
             ITokensLinesIterator tokens = processedDoc.GetProcessedTokensIterator();
             Token token = tokens.NextToken();
-            if (token != Token.END_OF_FILE)
+            if (token.TokenType != TokenType.EndOfFile)
             {
                 string documentPath = null;
                 int lineIndex = -1;
@@ -91,7 +91,7 @@ private static string ProcessTokensDocument(ProcessedTokensDocument processedDoc
                     }
                     sbTokens.AppendLine(token.ToString());
                 }
-                while ((token = tokens.NextToken()) != Token.END_OF_FILE);
+                while ((token = tokens.NextToken()).TokenType != TokenType.EndOfFile);
             }
 
             // Errors
diff --git a/TypeCobol/Compiler/AntlrUtils/AntlrPerformanceProfiler.cs b/TypeCobol/Compiler/AntlrUtils/AntlrPerformanceProfiler.cs
@@ -101,8 +101,8 @@ public void BeginParsingFile(TextSourceInfo textSourceInfo, ITokensLinesIterator
             if (tokensCountIterator != null)
             {
                 ITokensLine lastLine = null;
-                Token token = null;
-                while ((token = tokensCountIterator.NextToken()) != Token.END_OF_FILE)
+                Token token;
+                while ((token = tokensCountIterator.NextToken()).TokenType != TokenType.EndOfFile)
                 {
                     CurrentFileInfo.TokensCount++;
                     if (token.TokensLine != lastLine)
diff --git a/TypeCobol/Compiler/AntlrUtils/TokensLinesTokenStream.cs b/TypeCobol/Compiler/AntlrUtils/TokensLinesTokenStream.cs
@@ -61,15 +61,15 @@ public void SeekToToken(IToken searchedToken)
         }
 
         /// <summary>
-        /// Start monitoring if the token stream reached a specific token which marks the end of an instersting section
+        /// Start monitoring if the token stream reached a specific token which marks the end of an interesting section
         /// </summary>
         public void StartLookingForStopToken(Token stopToken)
         {
             ResetStopTokenLookup();
             if (stopToken != null)
             {
                 StopToken = stopToken;
-                stopTokenReplacedByEOF = new ReplacedToken(Token.END_OF_FILE, stopToken);
+                stopTokenReplacedByEOF = new ReplacedToken(Token.EndOfFile(), stopToken);
                 StreamReachedStopToken = false;
             }
         }
diff --git a/TypeCobol/Compiler/CupCommon/CobolWordsTokenizer.cs b/TypeCobol/Compiler/CupCommon/CobolWordsTokenizer.cs
@@ -233,7 +233,7 @@ public void LeaveAnyTokenMode(bool bAllMode = false)
         public void ConsumeNextTokenOnTheSameLine(TokenType nextTokenType)
         {
             Token currentToken = base.CurrentToken;
-            if (currentToken == Token.END_OF_FILE)
+            if (currentToken != null && currentToken.TokenType == TokenType.EndOfFile)
                 return;//Ignore if end of file
             Token nextToken = base.NextToken();
             if (nextToken != null && currentToken != null &&
@@ -258,21 +258,6 @@ public void ConsumeNextTokenOnTheSameLineAndStop(TokenType nextTokenType)
             EnterStopScanningMode();
         }
 
-        /// <summary>
-        /// Enter in the Stop Scanning Mode if the current is not of the given type
-        /// </summary>
-        /// <param name="tokenType">The Token type to check</param>
-        public void EnterStopScanningModeIfNotToken(TokenType tokenType)
-        {
-            Token currentToken = base.CurrentToken;
-            if (currentToken == null)
-                return;
-            if (currentToken == Token.END_OF_FILE)
-                return;//Ignore if end of file
-            if (currentToken.TokenType != tokenType)
-                EnterStopScanningMode();
-        }
-
         /// <summary>
         /// Enter in the Stop Scanning Mode if the next is not of the given type
         /// </summary>
@@ -282,7 +267,7 @@ public void EnterStopScanningModeIfNextNotToken(TokenType nextTokenType)
             Token currentToken = base.CurrentToken;
             if (currentToken == null)
                 return;
-            if (currentToken == Token.END_OF_FILE)
+            if (currentToken.TokenType == TokenType.EndOfFile)
                 return;//Ignore if end of file
             Token nextToken = base.NextToken();
             base.PreviousToken();
@@ -326,11 +311,12 @@ private bool BasicAnyTokenMode(Token token, Symbol symbol)
         /// </summary>
         /// <param name="expected">The expected next token</param>
         /// <param name="resulting">The resulting token if matching</param>
-        /// <param name="defaultToken">The defualt token if no matching</param>
+        /// <param name="defaultToken">The default token if no matching</param>
         /// <returns></returns>
         private int TryMatchNextToken(TokenType expected, int resulting, int defaultToken)
         {
-            if (base.CurrentToken == Token.END_OF_FILE)
+            var currentToken = base.CurrentToken;
+            if (currentToken != null && currentToken.TokenType == TokenType.EndOfFile)
                 return defaultToken;
             Token nextToken = base.NextToken();
             if (nextToken != null)
@@ -350,11 +336,11 @@ private int TryMatchNextToken(TokenType expected, int resulting, int defaultToke
         /// </summary>
         /// <param name="expected">The expected next token</param>
         /// <param name="resulting">The resulting token if matching</param>
-        /// <param name="defaultToken">The defualt token if no matching</param>
+        /// <param name="defaultToken">The default token if no matching</param>
         /// <returns></returns>
         private int TryMatchPrevToken(TokenType expected, int resulting, int defaultToken)
         {
-            if (base.CurrentToken == Token.END_OF_FILE)
+            if (base.CurrentToken == null)
                 return defaultToken;
             Token prevToken = base.PreviousToken();
             if (prevToken != null)
@@ -436,7 +422,7 @@ public IEnumerator<Symbol> GetEnumerator()
             FirstToken = null;
             LastToken = null;            
             Token token = null;
-            while ((token = base.NextToken()) != Token.END_OF_FILE)
+            while ((token = base.NextToken()).TokenType != TokenType.EndOfFile)
             {
                 if (FirstToken == null)
                 {
diff --git a/TypeCobol/Compiler/Parser/CodeElementsLinesTokenSource.cs b/TypeCobol/Compiler/Parser/CodeElementsLinesTokenSource.cs
@@ -103,7 +103,7 @@ public IToken NextToken()
                 }
             }
 
-            return Token.END_OF_FILE;
+            return Token.EndOfFile();
         }
 
         public string SourceName
diff --git a/TypeCobol/Compiler/Preprocessor/CopyTokensLinesIterator.cs b/TypeCobol/Compiler/Preprocessor/CopyTokensLinesIterator.cs
@@ -254,15 +254,16 @@ public Token NextToken()
             // If the document is empty or after end of file, immediately return EndOfFile
             if (currentLine == null)
             {
-                currentPosition.CurrentToken = Token.END_OF_FILE;
-                return Token.END_OF_FILE;
+                var eof = Token.EndOfFile();
+                currentPosition.CurrentToken = eof;
+                return eof;
             }
 
             // If the iterator is positioned in an imported document, return the next imported token
             if(currentPosition.ImportedDocumentIterator != null)
             {
                 Token nextImportedToken = currentPosition.ImportedDocumentIterator.NextToken();
-                if(nextImportedToken == Token.END_OF_FILE)
+                if(nextImportedToken.TokenType == TokenType.EndOfFile)
                 {
                     currentPosition.ImportedDocumentIterator = null;
                     currentPosition.ImportedDocumentIteratorPosition = null;
@@ -297,8 +298,9 @@ public Token NextToken()
                     {
                         // return EndOfFile
                         currentLine = null;
-                        currentPosition.CurrentToken = Token.END_OF_FILE;
-                        return Token.END_OF_FILE;
+                        var eof = Token.EndOfFile();
+                        currentPosition.CurrentToken = eof;
+                        return eof;
                     }
                 }
                 // Check if the next token found matches the filter criteria or is a COPY compiler directive or is a REPLACE directive
@@ -322,7 +324,7 @@ public Token NextToken()
 
                     // No suitable next token found in the imported document
                     // -> get next token in the main document
-                    if (nextTokenCandidate == Token.END_OF_FILE)
+                    if (nextTokenCandidate.TokenType == TokenType.EndOfFile)
                     {
                         return NextToken();
                     }
diff --git a/TypeCobol/Compiler/Scanner/ITokensLinesIterator.cs b/TypeCobol/Compiler/Scanner/ITokensLinesIterator.cs
@@ -51,7 +51,7 @@ public interface ITokensLinesIterator
         /// <summary>
         /// Get next token or EndOfFile
         /// </summary>
-            Token NextToken();
+        Token NextToken();
 
         /// <summary>
         /// Get null (before the first call to NextToken()), current token, or EndOfFile
diff --git a/TypeCobol/Compiler/Scanner/Token.cs b/TypeCobol/Compiler/Scanner/Token.cs
@@ -413,8 +413,14 @@ public int TokenIndex
             get { return -1; }
         }
 
-        // Common token for End of file
-        public static Token END_OF_FILE = new Token(TokenType.EndOfFile, 0, -1, TypeCobol.Compiler.Scanner.TokensLine.CreateVirtualLineForInsertedToken(-1, String.Empty));
+        /// <summary>
+        /// Creates a new instance of special end-of-file Token.
+        /// </summary>
+        /// <returns>New Token instance with EndOfFile TokenType</returns>
+        public static Token EndOfFile()
+        {
+            return new Token(TokenType.EndOfFile, 0, -1, Compiler.Scanner.TokensLine.CreateVirtualLineForInsertedToken(-1, string.Empty));
+        }
 
         // --- Token comparison for REPLACE directive ---
 
diff --git a/TypeCobol/Compiler/Scanner/TokensLinesIterator.cs b/TypeCobol/Compiler/Scanner/TokensLinesIterator.cs
@@ -199,7 +199,7 @@ public Token NextToken(bool applyChannelFilter)
             // If document is empty, immediately return EndOfFile
             if (currentLine == null)
             {
-                return Token.END_OF_FILE;
+                return Token.EndOfFile();
             }
 
             // While we can find a next token
@@ -224,7 +224,7 @@ public Token NextToken(bool applyChannelFilter)
                     {
                         // return EndOfFile
                         currentLine = null;
-                        return Token.END_OF_FILE;
+                        return Token.EndOfFile();
                     }
                 }
                 // Check if the next token found matches the filter criteria
diff --git a/TypeCobol/LanguageServices/CodeAnalysis/Statistics/StatsGenerator.cs b/TypeCobol/LanguageServices/CodeAnalysis/Statistics/StatsGenerator.cs
@@ -296,7 +296,7 @@ public static void GenerateStatisticsForPrograms(CompilationProject project, IEn
                         // Iterate over tokens AFTER preprocessing
                         ITokensLinesIterator processedTokensIterator = compilationResult.ProcessedTokensDocumentSnapshot.GetProcessedTokensIterator();
                         Token processedToken = null;
-                        while ((processedToken = processedTokensIterator.NextToken()) != Token.END_OF_FILE)
+                        while ((processedToken = processedTokensIterator.NextToken()).TokenType != TokenType.EndOfFile)
                         {
                             tokensCounter.OnElement((int)processedToken.TokenType);
                             ReplacedToken replacedToken = processedToken as ReplacedToken;

Original file line number	Diff line number	Diff line change
`@@ -45,7 +45,7 @@ public class Workspace`
`45`	`45`
`46`	`46`	`internal CompilationProject CompilationProject { get; private set; }`
`47`	`47`
`48`		`- private List<FileCompiler> _fileCompilerWaittingForNodePhase;`
	`48`	`+ private List<FileCompiler> _fileCompilerWaitingForNodePhase;`
`49`	`49`	`public TypeCobolConfiguration Configuration { get; private set; }`
`50`	`50`	`public event EventHandler<DiagnosticEvent> DiagnosticsEvent;`
`51`	`51`	`public event EventHandler<EventArgs> DocumentModifiedEvent;`
`@@ -162,7 +162,7 @@ public Workspace(string rootDirectoryFullName, string workspaceName, Queue<Messa`
`162`	`162`	`MessagesActionsQueue = messagesActionsQueue;`
`163`	`163`	`Configuration = new TypeCobolConfiguration();`
`164`	`164`	`_openedDocuments = new Dictionary<Uri, DocumentContext>();`
`165`		`- _fileCompilerWaittingForNodePhase = new List<FileCompiler>();`
	`165`	`+ _fileCompilerWaitingForNodePhase = new List<FileCompiler>();`
`166`	`166`	`_Logger = logger;`
`167`	`167`
`168`	`168`	`this._rootDirectoryFullName = rootDirectoryFullName;`
`@@ -234,22 +234,22 @@ private FileCompiler OpenTextDocument(DocumentContext docContext, string sourceT`
`234`	`234`	`#else`
`235`	`235`	`fileCompiler = new FileCompiler(initialTextDocumentLines, CompilationProject.SourceFileProvider, CompilationProject, CompilationProject.CompilationOptions, _customSymbols, CompilationProject);`
`236`	`236`	`#endif`
`237`		`- //Set Any Language Server Connection Options.`
`238`		`- docContext.FileCompiler = fileCompiler;`
`239`		`- docContext.LanguageServerConnection(true);`
`240`		`-`
`241`		`- fileCompiler.CompilationResultsForProgram.UpdateTokensLines();`
`242`	`237`
`243`	`238`	`lock (_lockForOpenedDocuments)`
`244`	`239`	`{`
`245`	`240`	`if (_openedDocuments.ContainsKey(docContext.Uri))`
`246`	`241`	`CloseSourceFile(docContext.Uri); //Close and remove the previous opened file.`
`247`	`242`
`248`	`243`	`_openedDocuments.Add(docContext.Uri, docContext);`
`249`		`- fileCompiler.CompilationResultsForProgram.CodeAnalysisCompleted += FinalCompilationStepCompleted;`
`250`	`244`	`}`
`251`	`245`
	`246`	`+ //Set Any Language Server Connection Options.`
	`247`	`+ docContext.FileCompiler = fileCompiler;`
	`248`	`+ docContext.LanguageServerConnection(true);`
	`249`	`+`
`252`	`250`	`fileCompiler.CompilationResultsForProgram.SetOwnerThread(Thread.CurrentThread);`
	`251`	`+ fileCompiler.CompilationResultsForProgram.CodeAnalysisCompleted += FinalCompilationStepCompleted;`
	`252`	`+ fileCompiler.CompilationResultsForProgram.UpdateTokensLines();`
`253`	`253`
`254`	`254`	`if (lsrOptions != LsrTestingOptions.LsrSourceDocumentTesting)`
`255`	`255`	`{`
`@@ -325,14 +325,14 @@ public void UpdateSourceFile(Uri fileUri, TextChangedEvent textChangedEvent)`
`325`	`325`	`{`
`326`	`326`	`if (!_timerDisabled) //If TimerDisabled is false, create a timer to automatically launch Node phase`
`327`	`327`	`{`
`328`		`- lock (_fileCompilerWaittingForNodePhase)`
	`328`	`+ lock (_fileCompilerWaitingForNodePhase)`
`329`	`329`	`{`
`330`		`- if (!_fileCompilerWaittingForNodePhase.Contains(fileCompilerToUpdate))`
`331`		`- _fileCompilerWaittingForNodePhase.Add(fileCompilerToUpdate); //Store that this fileCompiler will soon need a Node Phase`
	`330`	`+ if (!_fileCompilerWaitingForNodePhase.Contains(fileCompilerToUpdate))`
	`331`	`+ _fileCompilerWaitingForNodePhase.Add(fileCompilerToUpdate); //Store that this fileCompiler will soon need a Node Phase`
`332`	`332`	`}`
`333`	`333`
`334`	`334`	`_semanticUpdaterTimer = new System.Timers.Timer(750);`
`335`		`- _semanticUpdaterTimer.Elapsed += (sender, e) => TimerEvent(sender, e, fileCompilerToUpdate);`
	`335`	`+ _semanticUpdaterTimer.Elapsed += (sender, e) => TimerEvent(fileUri);`
`336`	`336`	`_semanticUpdaterTimer.Start();`
`337`	`337`	`}`
`338`	`338`	`}`
`@@ -392,23 +392,29 @@ private void ExecutionStepEventHandler(object oFileCompiler, ExecutionStepEventA`
`392`	`392`	`/// <param name="sender"></param>`
`393`	`393`	`/// <param name="eventArgs"></param>`
`394`	`394`	`/// <param name="fileCompiler"></param>`
`395`		`- private void TimerEvent(object sender, ElapsedEventArgs eventArgs, FileCompiler fileCompiler)`
	`395`	`+ private void TimerEvent(Uri fileUri)`
`396`	`396`	`{`
`397`	`397`	`try`
`398`	`398`	`{`
`399`	`399`	`_semanticUpdaterTimer.Stop();`
`400`		`- Action nodeRefreshAction = () => { RefreshSyntaxTree(fileCompiler, SyntaxTreeRefreshLevel.RebuildNodesAndPerformQualityCheck); };`
`401`	`400`	`lock (MessagesActionsQueue)`
`402`	`401`	`{`
`403`		`- MessagesActionsQueue.Enqueue(new MessageActionWrapper(nodeRefreshAction));`
	`402`	`+ MessagesActionsQueue.Enqueue(new MessageActionWrapper(Refresh));`
`404`	`403`	`}`
`405`	`404`	`}`
`406`	`405`	`catch (Exception e)`
`407`	`406`	`{`
`408`	`407`	`//In case Timer Thread crash`
`409`	`408`	`ExceptionTriggered(null, new ThreadExceptionEventArgs(e));`
`410`	`409`	`}`
`411`		`-`
	`410`	`+`
	`411`	`+ void Refresh()`
	`412`	`+ {`
	`413`	`+ if (TryGetOpenedDocumentContext(fileUri, out var docContext))`
	`414`	`+ {`
	`415`	`+ RefreshSyntaxTree(docContext.FileCompiler, SyntaxTreeRefreshLevel.RebuildNodesAndPerformQualityCheck);`
	`416`	`+ }`
	`417`	`+ }`
`412`	`418`	`}`
`413`	`419`
`414`	`420`	`/// <summary>`
`@@ -446,12 +452,12 @@ public void RefreshSyntaxTree(FileCompiler fileCompiler, SyntaxTreeRefreshLevel`
`446`	`452`	`{`
`447`	`453`	`if (refreshLevel == SyntaxTreeRefreshLevel.NoRefresh) return; //nothing to do`
`448`	`454`
`449`		`- lock (_fileCompilerWaittingForNodePhase)`
	`455`	`+ lock (_fileCompilerWaitingForNodePhase)`
`450`	`456`	`{`
`451`		`- var fileCompilerNeedsRefresh = _fileCompilerWaittingForNodePhase.Contains(fileCompiler);`
	`457`	`+ var fileCompilerNeedsRefresh = _fileCompilerWaitingForNodePhase.Contains(fileCompiler);`
`452`	`458`	`if (fileCompilerNeedsRefresh)`
`453`	`459`	`{`
`454`		`- _fileCompilerWaittingForNodePhase.Remove(fileCompiler);`
	`460`	`+ _fileCompilerWaitingForNodePhase.Remove(fileCompiler);`
`455`	`461`	`}`
`456`	`462`	`else`
`457`	`463`	`{`
`@@ -495,16 +501,27 @@ void RefreshCodeAnalysisResults()`
`495`	`501`	`/// </summary>`
`496`	`502`	`public void CloseSourceFile(Uri fileUri)`
`497`	`503`	`{`
	`504`	`+ FileCompiler fileCompilerToClose = null;`
	`505`	`+`
	`506`	`+ //Remove from opened documents dictionary`
`498`	`507`	`lock (_lockForOpenedDocuments)`
`499`	`508`	`{`
`500`		`- if (_openedDocuments.ContainsKey(fileUri))`
	`509`	`+ if (_openedDocuments.TryGetValue(fileUri, out var contextToClose))`
`501`	`510`	`{`
`502`		`- var contextToClose = _openedDocuments[fileUri];`
`503`		`- FileCompiler fileCompilerToClose = contextToClose.FileCompiler;`
	`511`	`+ fileCompilerToClose = contextToClose.FileCompiler;`
`504`	`512`	`_openedDocuments.Remove(fileUri);`
`505`	`513`	`fileCompilerToClose.CompilationResultsForProgram.CodeAnalysisCompleted -= FinalCompilationStepCompleted;`
`506`	`514`	`}`
`507`		`- }`
	`515`	`+ }`
	`516`	`+`
	`517`	`+ //Remove from pending semantic analysis list`
	`518`	`+ if (fileCompilerToClose != null)`
	`519`	`+ {`
	`520`	`+ lock (_fileCompilerWaitingForNodePhase)`
	`521`	`+ {`
	`522`	`+ _fileCompilerWaitingForNodePhase.Remove(fileCompilerToClose);`
	`523`	`+ }`
	`524`	`+ }`
`508`	`525`	`}`
`509`	`526`
`510`	`527`	`/// <summary>`
Original file line number	Diff line number	Diff line change
`@@ -73,7 +73,7 @@ private static string ProcessTokensDocument(ProcessedTokensDocument processedDoc`
`73`	`73`	`StringBuilder sbTokens = new StringBuilder();`
`74`	`74`	`ITokensLinesIterator tokens = processedDoc.GetProcessedTokensIterator();`
`75`	`75`	`Token token = tokens.NextToken();`
`76`		`- if (token != Token.END_OF_FILE)`
	`76`	`+ if (token.TokenType != TokenType.EndOfFile)`
`77`	`77`	`{`
`78`	`78`	`string documentPath = null;`
`79`	`79`	`int lineIndex = -1;`
`@@ -91,7 +91,7 @@ private static string ProcessTokensDocument(ProcessedTokensDocument processedDoc`
`91`	`91`	`}`
`92`	`92`	`sbTokens.AppendLine(token.ToString());`
`93`	`93`	`}`
`94`		`- while ((token = tokens.NextToken()) != Token.END_OF_FILE);`
	`94`	`+ while ((token = tokens.NextToken()).TokenType != TokenType.EndOfFile);`
`95`	`95`	`}`
`96`	`96`
`97`	`97`	`// Errors`
Original file line number	Diff line number	Diff line change
`@@ -101,8 +101,8 @@ public void BeginParsingFile(TextSourceInfo textSourceInfo, ITokensLinesIterator`
`101`	`101`	`if (tokensCountIterator != null)`
`102`	`102`	`{`
`103`	`103`	`ITokensLine lastLine = null;`
`104`		`- Token token = null;`
`105`		`- while ((token = tokensCountIterator.NextToken()) != Token.END_OF_FILE)`
	`104`	`+ Token token;`
	`105`	`+ while ((token = tokensCountIterator.NextToken()).TokenType != TokenType.EndOfFile)`
`106`	`106`	`{`
`107`	`107`	`CurrentFileInfo.TokensCount++;`
`108`	`108`	`if (token.TokensLine != lastLine)`
Original file line number	Diff line number	Diff line change
`@@ -61,15 +61,15 @@ public void SeekToToken(IToken searchedToken)`
`61`	`61`	`}`
`62`	`62`
`63`	`63`	`/// <summary>`
`64`		`- /// Start monitoring if the token stream reached a specific token which marks the end of an instersting section`
	`64`	`+ /// Start monitoring if the token stream reached a specific token which marks the end of an interesting section`
`65`	`65`	`/// </summary>`
`66`	`66`	`public void StartLookingForStopToken(Token stopToken)`
`67`	`67`	`{`
`68`	`68`	`ResetStopTokenLookup();`
`69`	`69`	`if (stopToken != null)`
`70`	`70`	`{`
`71`	`71`	`StopToken = stopToken;`
`72`		`- stopTokenReplacedByEOF = new ReplacedToken(Token.END_OF_FILE, stopToken);`
	`72`	`+ stopTokenReplacedByEOF = new ReplacedToken(Token.EndOfFile(), stopToken);`
`73`	`73`	`StreamReachedStopToken = false;`
`74`	`74`	`}`
`75`	`75`	`}`
Original file line number	Diff line number	Diff line change
`@@ -103,7 +103,7 @@ public IToken NextToken()`
`103`	`103`	`}`
`104`	`104`	`}`
`105`	`105`
`106`		`- return Token.END_OF_FILE;`
	`106`	`+ return Token.EndOfFile();`
`107`	`107`	`}`
`108`	`108`
`109`	`109`	`public string SourceName`
Original file line number	Diff line number	Diff line change
`@@ -254,15 +254,16 @@ public Token NextToken()`
`254`	`254`	`// If the document is empty or after end of file, immediately return EndOfFile`
`255`	`255`	`if (currentLine == null)`
`256`	`256`	`{`
`257`		`- currentPosition.CurrentToken = Token.END_OF_FILE;`
`258`		`- return Token.END_OF_FILE;`
	`257`	`+ var eof = Token.EndOfFile();`
	`258`	`+ currentPosition.CurrentToken = eof;`
	`259`	`+ return eof;`
`259`	`260`	`}`
`260`	`261`
`261`	`262`	`// If the iterator is positioned in an imported document, return the next imported token`
`262`	`263`	`if(currentPosition.ImportedDocumentIterator != null)`
`263`	`264`	`{`
`264`	`265`	`Token nextImportedToken = currentPosition.ImportedDocumentIterator.NextToken();`
`265`		`- if(nextImportedToken == Token.END_OF_FILE)`
	`266`	`+ if(nextImportedToken.TokenType == TokenType.EndOfFile)`
`266`	`267`	`{`
`267`	`268`	`currentPosition.ImportedDocumentIterator = null;`
`268`	`269`	`currentPosition.ImportedDocumentIteratorPosition = null;`
`@@ -297,8 +298,9 @@ public Token NextToken()`
`297`	`298`	`{`
`298`	`299`	`// return EndOfFile`
`299`	`300`	`currentLine = null;`
`300`		`- currentPosition.CurrentToken = Token.END_OF_FILE;`
`301`		`- return Token.END_OF_FILE;`
	`301`	`+ var eof = Token.EndOfFile();`
	`302`	`+ currentPosition.CurrentToken = eof;`
	`303`	`+ return eof;`
`302`	`304`	`}`
`303`	`305`	`}`
`304`	`306`	`// Check if the next token found matches the filter criteria or is a COPY compiler directive or is a REPLACE directive`
`@@ -322,7 +324,7 @@ public Token NextToken()`
`322`	`324`
`323`	`325`	`// No suitable next token found in the imported document`
`324`	`326`	`// -> get next token in the main document`
`325`		`- if (nextTokenCandidate == Token.END_OF_FILE)`
	`327`	`+ if (nextTokenCandidate.TokenType == TokenType.EndOfFile)`
`326`	`328`	`{`
`327`	`329`	`return NextToken();`
`328`	`330`	`}`