Merge pull request #56 from apex-dev-tools/fix-lexer-hex

pwrightcertinia · web-flow · commit a79a8aeff0a7 · 2024-11-14T10:26:53.000Z
Fix lexer hex characters
diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml
@@ -12,15 +12,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Set up Node 14
-        uses: actions/setup-node@v3
+      - name: Set up Node
+        uses: actions/setup-node@v4
         with:
-          node-version: 14
+          node-version: 20
 
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: "temurin"
           java-version: 8
@@ -30,12 +30,12 @@ jobs:
         run: npm run build
 
       - name: Checkout samples repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
           repository: apex-dev-tools/apex-samples
           path: apex-samples
-          ref: v1.0.2
+          ref: v1.2.0
 
       - name: Set samples env
         run: echo "SAMPLES=$GITHUB_WORKSPACE/apex-samples" >> "$GITHUB_ENV"
diff --git a/.github/workflows/PublishMaven.yml b/.github/workflows/PublishMaven.yml
@@ -2,26 +2,26 @@ name: Publish to Maven
 
 on:
   release:
-    types: [ released ]
+    types: [released]
   workflow_dispatch:
 
 jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Set up Node 14
-        uses: actions/setup-node@v3
+      - name: Set up Node
+        uses: actions/setup-node@v4
         with:
-          node-version: 14
+          node-version: 20
 
       - name: Set up Maven Central Repository
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
-          java-version: '8'
-          distribution: 'temurin'
-          cache: 'maven'
+          java-version: "8"
+          distribution: "temurin"
+          cache: "maven"
           server-id: ossrh
           server-username: MAVEN_USERNAME
           server-password: MAVEN_TOKEN
diff --git a/.github/workflows/PublishNPM.yml b/.github/workflows/PublishNPM.yml
@@ -2,20 +2,20 @@ name: Publish to NPM
 
 on:
   release:
-    types: [ released ]
+    types: [released]
   workflow_dispatch:
 
 jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Set up Node 14
-        uses: actions/setup-node@v3
+      - name: Set up Node
+        uses: actions/setup-node@v4
         with:
-          node-version: 14
-          registry-url: 'https://registry.npmjs.org'
+          node-version: 20
+          registry-url: "https://registry.npmjs.org"
 
       - name: Publish package
         run: |
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # apex-parser - Changelog
 
+## 4.3.1 - 2024-11-12
+
+- Fix Lexer support for uppercase Hex
+- Fix parser `whenValue` to support type refs
+
 ## 4.3.0 - 2024-09-26
 
 - Add `convertCurrency` and `FORMAT` SOQL/SOSL functions
diff --git a/README.md b/README.md
@@ -48,12 +48,12 @@ Maven
     <dependency>
         <groupId>io.github.apex-dev-tools</groupId>
         <artifactId>apex-parser</artifactId>
-        <version>4.3.0</version>
+        <version>4.3.1</version>
     </dependency>
 
 NPM
 
-    "@apexdevtools/apex-parser": "^4.3.0"
+    "@apexdevtools/apex-parser": "^4.3.1"
 
 ## Building
 
diff --git a/antlr/ApexLexer.g4 b/antlr/ApexLexer.g4
@@ -310,7 +310,7 @@ NumberLiteral
 
 fragment
 HexCharacter
-    :   Digit | 'a' | 'b' | 'c' | 'd' | 'e' | 'f'
+    :   [0-9a-fA-F]
     ;
 
 fragment
diff --git a/antlr/ApexParser.g4 b/antlr/ApexParser.g4
@@ -314,7 +314,7 @@ whenControl
 whenValue
     : ELSE
     | whenLiteral (COMMA whenLiteral)*
-    | id id
+    | typeRef id
     ;
 
 whenLiteral
diff --git a/jvm/pom.xml b/jvm/pom.xml
@@ -5,7 +5,7 @@
 
   <groupId>io.github.apex-dev-tools</groupId>
   <artifactId>apex-parser</artifactId>
-  <version>4.3.0</version>
+  <version>4.3.1</version>
   <packaging>jar</packaging>
 
   <name>apex-parser</name>
diff --git a/jvm/src/test/java/io/github/apexdevtools/apexparser/ApexLexerTest.java b/jvm/src/test/java/io/github/apexdevtools/apexparser/ApexLexerTest.java
@@ -13,42 +13,53 @@
  */
 package io.github.apexdevtools.apexparser;
 
-import org.antlr.v4.runtime.CharStreams;
-import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.*;
 import org.junit.jupiter.api.Test;
 
-import java.io.IOException;
-import java.io.StringReader;
+import java.util.Map;
 
+import static io.github.apexdevtools.apexparser.SyntaxErrorCounter.createLexer;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class ApexLexerTest {
 
     @Test
     void testLexerGeneratesTokens() {
-        ApexLexer lexer = new ApexLexer(CharStreams.fromString("public class Hello {}"));
-        CommonTokenStream tokens  = new CommonTokenStream(lexer);
+        Map.Entry<ApexLexer, SyntaxErrorCounter> lexerAndCounter = createLexer("public class Hello {}", false);
+        CommonTokenStream tokens  = new CommonTokenStream(lexerAndCounter.getKey());
         assertEquals(6, tokens.getNumberOfOnChannelTokens());
+        assertEquals(0, lexerAndCounter.getValue().getNumErrors());
     }
 
     @Test
-    void testCaseInsensitivityLowerCase() throws IOException {
-        ApexLexer lexer = new ApexLexer(new CaseInsensitiveInputStream(CharStreams.fromString("public")));
-        CommonTokenStream tokens  = new CommonTokenStream(lexer);
+    void testCaseInsensitivityLowerCase() {
+        Map.Entry<ApexLexer, SyntaxErrorCounter> lexerAndCounter = createLexer("public", true);
+        CommonTokenStream tokens  = new CommonTokenStream(lexerAndCounter.getKey());
         assertEquals(2, tokens.getNumberOfOnChannelTokens());
+        assertEquals(0, lexerAndCounter.getValue().getNumErrors());
     }
 
     @Test
-    void testCaseInsensitivityUpperCase() throws IOException {
-        ApexLexer lexer = new ApexLexer(new CaseInsensitiveInputStream(CharStreams.fromString("PUBLIC")));
-        CommonTokenStream tokens  = new CommonTokenStream(lexer);
+    void testCaseInsensitivityUpperCase() {
+        Map.Entry<ApexLexer, SyntaxErrorCounter> lexerAndCounter = createLexer("PUBLIC", true);
+        CommonTokenStream tokens  = new CommonTokenStream(lexerAndCounter.getKey());
         assertEquals(2, tokens.getNumberOfOnChannelTokens());
+        assertEquals(0, lexerAndCounter.getValue().getNumErrors());
     }
 
     @Test
-    void testCaseInsensitivityMixedCase() throws IOException {
-        ApexLexer lexer = new ApexLexer(new CaseInsensitiveInputStream(CharStreams.fromString("PuBliC")));
-        CommonTokenStream tokens  = new CommonTokenStream(lexer);
+    void testCaseInsensitivityMixedCase() {
+        Map.Entry<ApexLexer, SyntaxErrorCounter> lexerAndCounter = createLexer("PuBliC", true);
+        CommonTokenStream tokens  = new CommonTokenStream(lexerAndCounter.getKey());
         assertEquals(2, tokens.getNumberOfOnChannelTokens());
+        assertEquals(0, lexerAndCounter.getValue().getNumErrors());
+    }
+
+    @Test
+    void testLexerUnicodeEscapes() {
+        Map.Entry<ApexLexer, SyntaxErrorCounter> lexerAndCounter = createLexer("'Fran\\u00E7ois'", false);
+        CommonTokenStream tokens = new CommonTokenStream(lexerAndCounter.getKey());
+        assertEquals(2, tokens.getNumberOfOnChannelTokens());
+        assertEquals(0, lexerAndCounter.getValue().getNumErrors());
     }
 }
diff --git a/jvm/src/test/java/io/github/apexdevtools/apexparser/SyntaxErrorCounter.java b/jvm/src/test/java/io/github/apexdevtools/apexparser/SyntaxErrorCounter.java
@@ -36,6 +36,17 @@ public int getNumErrors() {
         return this.numErrors;
     }
 
+    public static Map.Entry<ApexLexer, SyntaxErrorCounter> createLexer(String input, Boolean caseInsensitive) {
+        CharStream stream = CharStreams.fromString(input);
+        ApexLexer lexer = new ApexLexer(caseInsensitive ? new CaseInsensitiveInputStream(stream) : stream);
+
+        lexer.removeErrorListeners();
+        SyntaxErrorCounter errorCounter = new SyntaxErrorCounter();
+        lexer.addErrorListener(errorCounter);
+
+        return new AbstractMap.SimpleEntry<>(lexer, errorCounter);
+    }
+
     public static Map.Entry<ApexParser, SyntaxErrorCounter> createParser(String input) {
         ApexLexer lexer = new ApexLexer(new CaseInsensitiveInputStream(CharStreams.fromString(input)));
         CommonTokenStream tokens = new CommonTokenStream(lexer);
@@ -45,6 +56,6 @@ public static Map.Entry<ApexParser, SyntaxErrorCounter> createParser(String inpu
         SyntaxErrorCounter errorCounter = new SyntaxErrorCounter();
         parser.addErrorListener(errorCounter);
 
-        return new AbstractMap.SimpleEntry<ApexParser, SyntaxErrorCounter>(parser, errorCounter);
+        return new AbstractMap.SimpleEntry<>(parser, errorCounter);
     }
 }
diff --git a/npm/.prettierrc b/npm/.prettierrc
@@ -0,0 +1,5 @@
+{
+  "trailingComma": "es5",
+  "arrowParens": "avoid",
+  "tabWidth": 4
+}
diff --git a/npm/package-lock.json b/npm/package-lock.json
diff --git a/npm/package.json b/npm/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@apexdevtools/apex-parser",
-  "version": "4.3.0",
+  "version": "4.3.1",
   "author": "Apex Dev Tools Team <apexdevtools@gmail.com> (https://github.com/apex-dev-tools)",
   "bugs": "https://github.com/apex-dev-tools/apex-parser/issues",
   "description": "Javascript parser for Salesforce Apex Language",
diff --git a/npm/snapshots/system/SampleParseSys.js.snap b/npm/snapshots/system/SampleParseSys.js.snap
diff --git a/npm/src/__tests__/ApexLexerTest.ts b/npm/src/__tests__/ApexLexerTest.ts
diff --git a/npm/src/__tests__/SyntaxErrorCounter.ts b/npm/src/__tests__/SyntaxErrorCounter.ts

Original file line number	Diff line number	Diff line change
`@@ -310,7 +310,7 @@ NumberLiteral`
`310`	`310`
`311`	`311`	`fragment`
`312`	`312`	`HexCharacter`
`313`		`- : Digit \| 'a' \| 'b' \| 'c' \| 'd' \| 'e' \| 'f'`
	`313`	`+ : [0-9a-fA-F]`
`314`	`314`	`;`
`315`	`315`
`316`	`316`	`fragment`
Original file line number	Diff line number	Diff line change
`@@ -314,7 +314,7 @@ whenControl`
`314`	`314`	`whenValue`
`315`	`315`	`: ELSE`
`316`	`316`	`\| whenLiteral (COMMA whenLiteral)*`
`317`		`- \| id id`
	`317`	`+ \| typeRef id`
`318`	`318`	`;`
`319`	`319`
`320`	`320`	`whenLiteral`