Adds feature to LabExtension to highlight valid MAGIC commands.

Author: Shushant Singh

src/jupyter_matlab_labextension/src/lezer-matlab/src/highlight.js

@@ -11,6 +11,7 @@ export const matlabHighlighting = styleTags({
     MultilineComment: t.comment,
     SystemCommand: t.meta,
     String: t.string,
+    Magic: t.monospace,
     '( )': t.paren,
     '[ ]': t.squareBracket,
     '{ }': t.brace

src/jupyter_matlab_labextension/src/lezer-matlab/src/matlab.grammar

@@ -8,9 +8,15 @@ expression {
   MultilineComment |
   keyword |
   Symbol |
-  SystemCommand
+  SystemCommand |
+  Magic
 }
 
+@external propSource matlabHighlighting from "./highlight"
+
+// Call out to comment parser. Since this is above the tokens block in this grammar, it takes precedence.
+@external tokens parseComments from "./parse_comments" { MultilineComment, LineComment, Magic }
+
 // See https://lezer.codemirror.net/docs/guide/ for documentation on syntax
 // specific to @tokens blocks, and how it differs from regular expression syntax.
 @tokens {
@@ -38,9 +44,4 @@ keyword {
 
 @skip { space | LineComment }
 
-@external propSource matlabHighlighting from "./highlight"
-
-// Call out to comment parser.
-@external tokens parseComments from "./parse_comments" { MultilineComment, LineComment }
-
 @detectDelim

src/jupyter_matlab_labextension/src/lezer-matlab/src/parse_comments.js

@@ -2,7 +2,7 @@
 
 import { ExternalTokenizer } from '@lezer/lr';
 // This file is created by lezer-generator during the build.
-import { MultilineComment, LineComment } from './parser.terms.js';
+import { MultilineComment, LineComment, Magic } from './parser.terms.js';
 
 const percent = '%'.charCodeAt(0);
 const openBrace = '{'.charCodeAt(0);
@@ -12,6 +12,8 @@ const fileEnd = -1;
 const newline = '\n'.charCodeAt(0);
 const carriageReturn = '\r'.charCodeAt(0);
 
+const isAlphabetical = (char) => /^[a-zA-Z]$/.test(String.fromCharCode(char));
+
 const lineEndArray = [newline, carriageReturn, fileEnd, fileStart];
 
 const isWhitespace = (char) => /\s/.test(char);
@@ -84,33 +86,75 @@ const validMultiLineCommentEnd = (input) => {
     return true;
 };
 
-export const parseComments = new ExternalTokenizer(input => {
-    // First, check if it's just a line comment.
+const validMagic = (input) => {
+    if (input.notMagic !== undefined) {
+        return false;
+    }
+    var isMagic = false;
+    if (
+        input.peek(0) === percent &&
+        input.peek(1) === percent &&
+        isAlphabetical(input.peek(2))
+    ) {
+        isMagic = true;
+    }
+    return isMagic;
+};
+
+export const parseComments = new ExternalTokenizer((input) => {
+    // Tokenize only if the line is a comment, multiline comment
+    // or a magic and starts with a percentage.
     if (input.peek(0) !== percent) {
+        // If the line starts with anything other than a percentage then it is MATLAB Code.
+        // If the input.input.string exists then check it's length otherwise ignore the keys by returning true.
+        if (
+            input.peek(0) !== fileStart && input.peek(0) !== newline &&
+            (
+                !('input' in input) ||
+                !('string' in input.input) ||
+                input.input.string.length !== 0
+            )
+        ) {
+            input.notMagic = true;
+        }
         return;
-    } else if (!validMultiLineCommentStart(input)) {
-        while (!lineEndArray.includes(input.peek(0))) { input.advance(1); }
-        input.acceptToken(LineComment);
+    } else if (validMagic(input)) {
+        while (!lineEndArray.includes(input.peek(0))) {
+            input.advance(1);
+        }
+        input.acceptToken(Magic);
         return;
-    }
-    // Consume the %{
-    input.advance(2);
+    } else if (validMultiLineCommentStart(input)) {
+        // Consume the %{
+        input.advance(2);
+        // Multiline comments are treated as MATLAB Code.
+        input.notMagic = true;
+        // Now we know we've started a multiline comment, so
+        // continue until the end of the input or until the comment is closed.
+        // We need to keep track of the depth of nested multiline comments.
+        let depth = 1;
+        while (input.peek(0) !== fileEnd) {
+            if (validMultiLineCommentEnd(input)) {
+                input.advance(2);
+                depth--;
+                if (depth === 0) {
+                    break;
+                }
+            } else if (validMultiLineCommentStart(input)) {
+                depth++;
+            }
+            input.advance(1);
+        }
 
-    // Now we know we've started a multiline comment, so
-    // continue until the end of the input or until the comment is closed.
-    // We need to keep track of the depth of nested multiline comments.
-    let depth = 1;
-    while (input.peek(0) !== fileEnd) {
-        if (validMultiLineCommentEnd(input)) {
-            input.advance(2);
-            depth--;
-            if (depth === 0) { break; }
-        } else if (validMultiLineCommentStart(input)) {
-            depth++;
+        // Emit the token for the entire multiline comment
+        input.acceptToken(MultilineComment);
+    } else {
+        // Comments are also treated as MATLAB Code.
+        input.notMagic = true;
+        while (!lineEndArray.includes(input.peek(0))) {
+            input.advance(1);
         }
-        input.advance(1);
+        input.acceptToken(LineComment);
+        return;
     }
-
-    // Emit the token for the entire multiline comment
-    input.acceptToken(MultilineComment);
 });

src/jupyter_matlab_labextension/src/lezer-matlab/test/additional_cases.txt

@@ -56,3 +56,34 @@ test!test
 ==>
 
 Script(Identifier,SystemCommand)
+
+# Magic command after a Line Comment is treated as a Line Comment
+
+%%magic command
+% A comment
+%%magic now treated as comment
+
+==>
+
+Script(Magic, LineComment, LineComment)
+
+# Magic command after a MultilineComment is treated as a LineComment
+
+%{
+A MultilineComment
+%}
+%%magic now treated as comment
+
+==>
+
+Script(MultilineComment, LineComment)
+
+# Magic command after a MATLAB Code should be treated as a Line Comment
+
+%%magic command1
+peaks
+%%magic command2
+
+==>
+
+Script(Magic, Identifier, LineComment)

src/jupyter_matlab_labextension/src/lezer-matlab/test/basic_terms.txt

@@ -71,3 +71,11 @@ test ! echo "Test system command"
 ==>
 
 Script(Identifier,SystemCommand)
+
+# Magic command
+
+%%magic command
+
+==>
+
+Script(Magic)

src/jupyter_matlab_labextension/src/lezer-matlab/test/nested_terms.txt

@@ -68,3 +68,14 @@ Script(LineComment,MultilineComment)
 ==>
 
 Script(String,Symbol,String,Symbol)
+
+# Nested Magic commands
+
+%%magic command 1
+%%magic command 2
+
+%%magic command 3
+
+==>
+
+Script(Magic, Magic, Magic)