Expansion: Allow for identifying Log output as Semantic

.gitignore

@@ -39,3 +39,5 @@ jspm_packages
 *.vsix
 
 out
+
+tsconfig.tsbuildinfo

.vscode/launch.json

@@ -8,7 +8,18 @@
             "runtimeExecutable": "${execPath}",
             "args": [
                 "--extensionDevelopmentPath=${workspaceRoot}"
-            ]
-        }
+            ],
+        },
+        {
+			"name": "Run Client Extension Tests",
+			"type": "extensionHost",
+			"request": "launch",
+			"runtimeExecutable": "${execPath}",
+			"args": [
+				"--extensionDevelopmentPath=${workspaceFolder}",
+				"--extensionTestsPath=${workspaceFolder}/client/out/test/suite/index"
+			],
+			"outFiles": ["${workspaceFolder}/client/out/test/**/*.js"]
+		}
     ]
 }

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "debug.node.autoAttach": "on"
+  }

README.md

@@ -10,6 +10,68 @@ Language extension for VSCode/Bluemix Code that adds syntax colorization for bot
 
 Colorization should work with most themes because it uses common theme token style names. It also works with most instances of the output panel. Initially attempts to match common literals (strings, dates, numbers, guids) and warning|info|error|server|local messages.
 
+# Customization
+
+You can customize the semantic identification of the output by adding entries to outputcolorizer.semantics.
+
+## Example:
+
+In settings.json:
+
+```json
+    "outputcolorizer.semantics": [
+        {
+            "pattern": "{.*}",
+            "applies": "between",
+            "tokenType": "info",
+            "tokenModifier": "output"
+        },
+    ]
+```
+
+This will identify anything inside the matching regex pattern as info.output semantics.  Now you are free to add your own color rule:
+
+```json
+// settings.json
+    "editor.semanticTokenColorCustomizations": {
+        "rules": {
+            "error.output": "#ff0000",
+            "info.output": "#00ff00"
+        },
+    },
+```
+
+## Settings Object:
+
+```typescript
+interface SemanticObject {
+  pattern: string;    // A Regex compliant string
+  applies: 'between' | 'eol' | 'eof';
+  tokenType: string;
+  tokenModifier: string;
+}
+```
+
+* pattern
+
+  This is a regex string fed into RegExp() class.  Escape rules apply just as they would to the RegExp class in javascript.
+
+* applies
+
+    | Applies | Description |
+    | ------- | ----------- |
+    | between | From the start to the end of the regex will be identified by the semantic tokenType.tokenModifier |
+    | eol     | From the start of the regex match to the end of the line will be identified by the sematnic tokenType.tokenModifier, unless another token occurs before then |
+    | eof     | From the start of the regex match to the end of the file will be identified by the sematnic tokenType.tokenModifier, unless another token occurs before then |
+
+* tokenType
+
+    This can be either a [standard token](https://code.visualstudio.com/api/language-extensions/semantic-highlight-guide#standard-token-types-and-modifiers) or a custom name that you would like to assign values to separately.
+
+* tokenModifier
+
+     This can be either a [standard modifier](https://code.visualstudio.com/api/language-extensions/semantic-highlight-guide#standard-token-types-and-modifiers) or a custom name that you would like to assign values to separately.
+
 ## Change Log
 
 * **0.1.2** - Updated for compliance with upcoming VS Code marketplace changes

client/package.json

@@ -0,0 +1,25 @@
+{
+    "name": "output-colorizer-client",
+    "description": "Language Server Client for starting the output-colorizer server",
+    "version": "0.1.0",
+    "author": "Justin Hanselman",
+    "engines": {
+        "vscode": "^1.52.0"
+    },
+    "scripts": {
+        "watch": "tsc -watch -p ./"
+    },
+    "dependencies": {},
+    "devDependencies": {
+        "@types/chai": "^4.2.21",
+        "@types/glob": "^7.1.4",
+        "@types/mocha": "^9.0.0",
+        "@types/node": "^16.4.13",
+        "@types/vscode": "^1.59.0",
+        "@vscode/test-electron": "^1.6.2",
+        "chai": "^4.3.4",
+        "glob": "^7.1.7",
+        "mocha": "^9.0.3",
+        "typescript": "^4.3.5"
+    }
+}

client/src/extension.ts

@@ -0,0 +1,157 @@
+import {
+    ExtensionContext,
+    workspace,
+    languages,
+    TextDocument,
+    Position,
+    ProviderResult,
+    Range,
+    SemanticTokensLegend,
+    DocumentSemanticTokensProvider,
+    SemanticTokens,
+    SemanticTokensBuilder,
+    CancellationToken,
+    Disposable,
+} from 'vscode';
+
+export interface SemanticObject {
+  // A Regex compliant string
+  pattern: string;
+  applies: 'between' | 'eol' | 'eof',
+  tokenType: string;
+  tokenModifier: string;
+}
+
+interface SemanticsFounds {
+  // The index of the identifier in the SemanticsProvider
+  idIdx: number;
+  // The start index of the semantic match token
+  matchStartIdx: number;
+  matchEndIdx: number;
+}
+
+export class OutputColorizerSemanticsProvider implements DocumentSemanticTokensProvider {
+
+  private identifiers: SemanticObject[];
+  legend: SemanticTokensLegend;
+
+  constructor(identifierConfig: SemanticObject[]) {
+    this.identifiers = identifierConfig;
+    const tokenTypes: string[] = [];
+    const tokenModifiers: string[] = [];
+    identifierConfig.forEach((iConfig) => {
+      tokenTypes.push(iConfig.tokenType);
+      tokenModifiers.push(iConfig.tokenModifier);
+    });
+    this.legend = new SemanticTokensLegend(tokenTypes, tokenModifiers);
+  }
+
+  provideDocumentSemanticTokens(doc: TextDocument, token: CancellationToken): ProviderResult<SemanticTokens> {
+    // analyze the document and return semantic tokens
+
+    const tokensBuilder = new SemanticTokensBuilder(this.legend);
+    const text = doc.getText();
+
+    // For now, not sure of async behavior, so we make Regexp every time
+    const regexps = this.identifiers.map(i => RegExp(i.pattern, "g"));
+
+    // Get all regexes against the document
+    const semanticsFoundMap : SemanticsFounds[] = [];
+    for (let i = 0; i < regexps.length; i++) {
+      let match = null;
+      const regexp = regexps[i];
+      do {
+        match = null;
+        match = regexp.exec(text);
+        if (match) {
+          semanticsFoundMap.push({
+            idIdx: i,
+            matchEndIdx: regexp.lastIndex,
+            matchStartIdx: regexp.lastIndex - match[0].length,
+          });
+        }
+      } while(match)
+    }
+
+    // sort by startIdx
+    semanticsFoundMap.sort((a, b) => a.matchStartIdx - b.matchStartIdx)
+      .forEach((found, idx) => {
+        const identifier = this.identifiers[found.idIdx];
+        const startPos = doc.positionAt(found.matchStartIdx);
+        let nextSemantic : Position | null = null;
+        let endPosition : Position | null = null;
+        switch(identifier.applies) {
+          case "between":
+            tokensBuilder.push(new Range(
+              startPos,
+              doc.positionAt(found.matchEndIdx),
+            ), identifier.tokenType, [identifier.tokenModifier]);
+            break;
+          case "eol":
+            const line = doc.lineAt(startPos.line)
+            if (idx < semanticsFoundMap.length - 1) {
+              nextSemantic = doc.positionAt(semanticsFoundMap[idx + 1].matchStartIdx);
+            }
+
+            if (nextSemantic && nextSemantic.line === line.lineNumber) {
+              endPosition = new Position(line.lineNumber, nextSemantic.character);
+            } else {
+              endPosition = new Position(line.lineNumber, line.range.end.character);
+            }
+            tokensBuilder.push(new Range(
+              startPos,
+              endPosition,
+            ), identifier.tokenType, [identifier.tokenModifier]);
+            break;
+          case 'eof':
+            const eof = doc.positionAt(text.length - 1);
+            if (idx < semanticsFoundMap.length - 1) {
+              nextSemantic = doc.positionAt(semanticsFoundMap[idx + 1].matchStartIdx)
+            }
+
+            for(let curLine = startPos.line, end = false; !end && curLine <= eof.line; curLine++) {
+              if (nextSemantic && nextSemantic.line === curLine) {
+                endPosition = new Position(curLine, nextSemantic.character);
+                end = true;
+              } else {
+                const curLineObj = doc.lineAt(curLine);
+                endPosition = new Position(curLine, curLineObj.range.end.character);
+              }
+              tokensBuilder.push(new Range(
+                curLine == startPos.line ? startPos : new Position(curLine, 0),
+                endPosition,
+              ), identifier.tokenType, [identifier.tokenModifier]);
+            }
+            break;
+        }
+      });
+
+    return tokensBuilder.build();
+  }
+}
+
+let providerDisposable : Disposable;
+export function activate(context: ExtensionContext) {
+
+  const semantics = workspace.getConfiguration().get<SemanticObject[]>('outputcolorizer.semantics');
+  const provider = new OutputColorizerSemanticsProvider(semantics ?? []);
+  providerDisposable = languages.registerDocumentSemanticTokensProvider('Log', provider, provider.legend);
+
+  context.subscriptions.push(workspace.onDidChangeConfiguration(e => {
+    if (e.affectsConfiguration('outputcolorizer.semantics')) {
+      // Dispose of the old semantics provider
+      providerDisposable.dispose();
+
+      const semantics = workspace.getConfiguration().get<SemanticObject[]>('outputcolorizer.semantics');
+      const provider = new OutputColorizerSemanticsProvider(semantics ?? []);
+      providerDisposable = languages.registerDocumentSemanticTokensProvider('Log', provider, provider.legend);
+    }
+  }));
+}
+
+export function deactivate(): Thenable<void> | undefined {
+  if (!providerDisposable) {
+    return undefined;
+  }
+  return providerDisposable.dispose();
+}

client/src/test/runTest.ts

@@ -0,0 +1,23 @@
+import * as path from 'path';
+
+import { runTests } from '@vscode/test-electron';
+
+async function main() {
+	try {
+		// The folder containing the Extension Manifest package.json
+		// Passed to `--extensionDevelopmentPath`
+		const extensionDevelopmentPath = path.resolve(__dirname, '../../');
+
+		// The path to the extension test script
+		// Passed to --extensionTestsPath
+		const extensionTestsPath = path.resolve(__dirname, './suite/index');
+
+		// Download VS Code, unzip it and run the integration test
+		await runTests({ extensionDevelopmentPath, extensionTestsPath });
+	} catch (err) {
+		console.error('Failed to run tests');
+		process.exit(1);
+	}
+}
+
+main();