stricter identifier parsing

MagnusOpera · Apr 27, 2024 · fe1990c · fe1990c
1 parent d70b8f8
commit fe1990c
Show file tree

Hide file tree

Showing 10 changed files with 1,169 additions and 957 deletions.
diff --git a/src/Terrabuild.Configuration/Gen/ProjectLexer.fs b/src/Terrabuild.Configuration/Gen/ProjectLexer.fs
diff --git a/src/Terrabuild.Configuration/Gen/ProjectParser.fs b/src/Terrabuild.Configuration/Gen/ProjectParser.fs
diff --git a/src/Terrabuild.Configuration/Gen/ProjectParser.fsi b/src/Terrabuild.Configuration/Gen/ProjectParser.fsi
@@ -31,6 +31,8 @@ type token =
   | NUMBER of (int)
   | KEY of (string)
   | VARIABLE of (string)
+  | TARGET_IDENTIFIER of (string)
+  | EXTENSION_IDENTIFIER of (string)
   | IDENTIFIER of (string)
   | STRING of (string)
   | NOTHING
@@ -67,6 +69,8 @@ type tokenId =
     | TOKEN_NUMBER
     | TOKEN_KEY
     | TOKEN_VARIABLE
+    | TOKEN_TARGET_IDENTIFIER
+    | TOKEN_EXTENSION_IDENTIFIER
     | TOKEN_IDENTIFIER
     | TOKEN_STRING
     | TOKEN_NOTHING
@@ -99,10 +103,12 @@ type nonTerminalId =
     | NONTERM_String
     | NONTERM_ListOfString
     | NONTERM_Strings
-    | NONTERM_ListOfIdentifiers
-    | NONTERM_Identifiers
+    | NONTERM_ListOfTargetIdentifiers
+    | NONTERM_TargetIdentifiers
     | NONTERM_Variables
     | NONTERM_Variable
+    | NONTERM_TargetIdentifier
+    | NONTERM_ExtensionIdentifier
     | NONTERM_Expr
 /// This function maps tokens to integer indexes
 val tagOfToken: token -> int

diff --git a/src/Terrabuild.Configuration/Gen/WorkspaceLexer.fs b/src/Terrabuild.Configuration/Gen/WorkspaceLexer.fs
diff --git a/src/Terrabuild.Configuration/Gen/WorkspaceParser.fs b/src/Terrabuild.Configuration/Gen/WorkspaceParser.fs
diff --git a/src/Terrabuild.Configuration/Gen/WorkspaceParser.fsi b/src/Terrabuild.Configuration/Gen/WorkspaceParser.fsi
@@ -30,6 +30,8 @@ type token =
   | NUMBER of (int)
   | KEY of (string)
   | VARIABLE of (string)
+  | TARGET_IDENTIFIER of (string)
+  | EXTENSION_IDENTIFIER of (string)
   | IDENTIFIER of (string)
   | STRING of (string)
   | NOTHING
@@ -65,6 +67,8 @@ type tokenId =
     | TOKEN_NUMBER
     | TOKEN_KEY
     | TOKEN_VARIABLE
+    | TOKEN_TARGET_IDENTIFIER
+    | TOKEN_EXTENSION_IDENTIFIER
     | TOKEN_IDENTIFIER
     | TOKEN_STRING
     | TOKEN_NOTHING
@@ -95,12 +99,14 @@ type nonTerminalId =
     | NONTERM_String
     | NONTERM_ListOfString
     | NONTERM_Strings
-    | NONTERM_ListOfIdentifiers
-    | NONTERM_Identifiers
+    | NONTERM_ListOfTargetIdentifiers
+    | NONTERM_TargetIdentifiers
     | NONTERM_StringVariables
     | NONTERM_StringVariable
     | NONTERM_Variables
     | NONTERM_Variable
+    | NONTERM_TargetIdentifier
+    | NONTERM_ExtensionIdentifier
     | NONTERM_Expr
 /// This function maps tokens to integer indexes
 val tagOfToken: token -> int

diff --git a/src/Terrabuild.Configuration/ProjectParser/Lexer.fsl b/src/Terrabuild.Configuration/ProjectParser/Lexer.fsl
@@ -8,7 +8,9 @@ let lexeme = LexBuffer<_>.LexemeString
 }
 
 let doubleQuote = '\"'
-let identifier = ['@' '^']? ['a'-'z'] ('_'? ['a'-'z' '0'-'9']+)*
+let extensionIdentifier = '@'? ['a'-'z'] ('_'? ['a'-'z' '0'-'9']+)*
+let targetIdentifier = '^'? ['a'-'z'] ('_'? ['a'-'z' '0'-'9']+)*
+let identifier = ['a'-'z'] ('_'? ['a'-'z' '0'-'9']+)*
 let key = identifier ' '* ':'
 let variable = '$' identifier
 let quotedString = '\"' [^'\"']+ '\"'
@@ -43,6 +45,8 @@ rule token = parse
     | "//" { singleLineComment lexbuf }
 
     | identifier { IDENTIFIER (lexeme lexbuf |> string) }
+    | extensionIdentifier { EXTENSION_IDENTIFIER (lexeme lexbuf |> string) }
+    | targetIdentifier { TARGET_IDENTIFIER (lexeme lexbuf |> string) }
 
     | quotedString { 
         let s = lexeme lexbuf |> string

diff --git a/src/Terrabuild.Configuration/ProjectParser/Parser.fsy b/src/Terrabuild.Configuration/ProjectParser/Parser.fsy
@@ -18,6 +18,8 @@ let debugPrint s = ignore s
 %token NOTHING TRUE FALSE
 %token <string> STRING
 %token <string> IDENTIFIER
+%token <string> EXTENSION_IDENTIFIER
+%token <string> TARGET_IDENTIFIER
 %token <string> VARIABLE
 %token <string> KEY
 %token <int> NUMBER
@@ -50,7 +52,7 @@ ProjectComponents:
 
 
 Extension:
-    | EXTENSION IDENTIFIER LBRACE ExtensionComponents RBRACE { ProjectComponents.Extension ($2, $4) }
+    | EXTENSION ExtensionIdentifier LBRACE ExtensionComponents RBRACE { ProjectComponents.Extension ($2, $4) }
 ExtensionComponents:
     | /* empty */ { Extension.Empty }
     | ExtensionComponents ExtensionContainer { $1.Patch $2 }
@@ -66,8 +68,8 @@ ExtensionDefaults:
 
 Configuration:
     | CONFIGURATION LBRACE ConfigurationComponents RBRACE { ProjectComponents.Configuration $3 }
-    | CONFIGURATION IDENTIFIER LBRACE ConfigurationComponents RBRACE { ConfigurationComponents.Init $2 |> $4.Patch |> ProjectComponents.Configuration }
-    | CONFIGURATION IDENTIFIER { ConfigurationComponents.Init $2 |> Configuration.Empty.Patch |> ProjectComponents.Configuration }
+    | CONFIGURATION ExtensionIdentifier LBRACE ConfigurationComponents RBRACE { ConfigurationComponents.Init $2 |> $4.Patch |> ProjectComponents.Configuration }
+    | CONFIGURATION ExtensionIdentifier { ConfigurationComponents.Init $2 |> Configuration.Empty.Patch |> ProjectComponents.Configuration }
 ConfigurationComponents:
     | /* empty */ { Configuration.Empty }
     | ConfigurationComponents ConfigurationDependencies { $1.Patch $2 }
@@ -93,14 +95,14 @@ TargetComponents:
     | TargetComponents TargetOutputs { $1.Patch $2 }
     | TargetComponents TargetStep { $1.Patch $2 }
 TargetDependsOn:
-    | DEPENDS_ON EQUAL ListOfIdentifiers { TargetComponents.DependsOn $3 }
+    | DEPENDS_ON EQUAL ListOfTargetIdentifiers { TargetComponents.DependsOn $3 }
 TargetRebuild:
     | REBUILD EQUAL Bool { TargetComponents.Rebuild $3 }
 TargetOutputs:
     | OUTPUTS EQUAL ListOfString { TargetComponents.Outputs $3 }
 TargetStep:
-    | IDENTIFIER IDENTIFIER { TargetComponents.Step { Extension = $1; Command = $2; Parameters = Map.empty } }
-    | IDENTIFIER IDENTIFIER LBRACE Variables RBRACE { TargetComponents.Step { Extension = $1; Command = $2; Parameters = $4 } }
+    | ExtensionIdentifier IDENTIFIER { TargetComponents.Step { Extension = $1; Command = $2; Parameters = Map.empty } }
+    | ExtensionIdentifier IDENTIFIER LBRACE Variables RBRACE { TargetComponents.Step { Extension = $1; Command = $2; Parameters = $4 } }
 
 Bool:
     | TRUE { true }
@@ -114,18 +116,26 @@ Strings:
     | /* empty */ { [] }
     | Strings String { $1 @ [$2] }
 
-ListOfIdentifiers:
-    | LSQBRACKET Identifiers RSQBRACKET { $2 }
-Identifiers:
+ListOfTargetIdentifiers:
+    | LSQBRACKET TargetIdentifiers RSQBRACKET { $2 }
+TargetIdentifiers:
     | /* empty */ { [] }
-    | Identifiers IDENTIFIER { $1 @ [$2] }
+    | TargetIdentifiers TargetIdentifier { $1 @ [$2] }
 
 Variables:
     | /* empty */ { Map.empty }        
     | Variables Variable { $1.Add $2 }
 Variable:
     | KEY Expr { ($1, $2) }
 
+TargetIdentifier:
+    | TARGET_IDENTIFIER { $1 }
+    | IDENTIFIER { $1 }
+
+ExtensionIdentifier:
+    | EXTENSION_IDENTIFIER { $1 }
+    | IDENTIFIER { $1 }
+
 Expr:
     | NOTHING { Expr.Nothing }
     | TRUE { Expr.Boolean true }

diff --git a/src/Terrabuild.Configuration/WorkspaceParser/Lexer.fsl b/src/Terrabuild.Configuration/WorkspaceParser/Lexer.fsl
@@ -8,7 +8,9 @@ let lexeme = LexBuffer<_>.LexemeString
 }
 
 let doubleQuote = '\"'
-let identifier = ['@' '^']? ['a'-'z'] ('_'? ['a'-'z' '0'-'9']+)*
+let extensionIdentifier = '@'? ['a'-'z'] ('_'? ['a'-'z' '0'-'9']+)*
+let targetIdentifier = '^'? ['a'-'z'] ('_'? ['a'-'z' '0'-'9']+)*
+let identifier = ['a'-'z'] ('_'? ['a'-'z' '0'-'9']+)*
 let key = identifier ' '* ':'
 let variable = '$' identifier
 let quotedString = '\"' [^'\"']+ '\"'
@@ -53,6 +55,8 @@ rule token = parse
     | "//" { singleLineComment lexbuf }
 
     | identifier { IDENTIFIER (lexeme lexbuf |> string) }
+    | extensionIdentifier { EXTENSION_IDENTIFIER (lexeme lexbuf |> string) }
+    | targetIdentifier { TARGET_IDENTIFIER (lexeme lexbuf |> string) }
 
     | quotedString { 
         let s = lexeme lexbuf |> string

diff --git a/src/Terrabuild.Configuration/WorkspaceParser/Parser.fsy b/src/Terrabuild.Configuration/WorkspaceParser/Parser.fsy
@@ -18,6 +18,8 @@ let debugPrint s = ignore s
 %token NOTHING TRUE FALSE
 %token <string> STRING
 %token <string> IDENTIFIER
+%token <string> EXTENSION_IDENTIFIER
+%token <string> TARGET_IDENTIFIER
 %token <string> VARIABLE
 %token <string> KEY
 %token <int> NUMBER
@@ -65,7 +67,7 @@ TargetComponents:
     | TargetComponents TargetDependsOn { $1.Patch $2 }
     | TargetComponents TargetRebuild { $1.Patch $2 }
 TargetDependsOn:
-    | DEPENDS_ON EQUAL ListOfIdentifiers { TargetComponents.DependsOn $3 }
+    | DEPENDS_ON EQUAL ListOfTargetIdentifiers { TargetComponents.DependsOn $3 }
 TargetRebuild:
     | REBUILD EQUAL Bool { TargetComponents.Rebuild $3 }
 
@@ -79,7 +81,7 @@ EnvironmentVariables:
     | VARIABLES EQUAL LBRACE Variables RBRACE { EnvironmentComponents.Variables $4 }
 
 Extension:
-    | EXTENSION IDENTIFIER LBRACE ExtensionComponents RBRACE { WorkspaceComponents.Extension ($2, $4) }
+    | EXTENSION ExtensionIdentifier LBRACE ExtensionComponents RBRACE { WorkspaceComponents.Extension ($2, $4) }
 ExtensionComponents:
     | /* empty */ { Extension.Empty }
     | ExtensionComponents ExtensionContainer { $1.Patch $2 }
@@ -104,11 +106,11 @@ Strings:
     | /* empty */ { [] }
     | Strings String { $1 @ [$2] }
 
-ListOfIdentifiers:
-    | LSQBRACKET Identifiers RSQBRACKET { $2 }
-Identifiers:
+ListOfTargetIdentifiers:
+    | LSQBRACKET TargetIdentifiers RSQBRACKET { $2 }
+TargetIdentifiers:
     | /* empty */ { [] }
-    | Identifiers IDENTIFIER { $1 @ [$2] }
+    | TargetIdentifiers TargetIdentifier { $1 @ [$2] }
 
 StringVariables:
     | /* empty */ { Map.empty }
@@ -122,6 +124,14 @@ Variables:
 Variable:
     | KEY Expr { ($1, $2) }
 
+TargetIdentifier:
+    | TARGET_IDENTIFIER { $1 }
+    | IDENTIFIER { $1 }
+
+ExtensionIdentifier:
+    | EXTENSION_IDENTIFIER { $1 }
+    | IDENTIFIER { $1 }
+
 Expr:
     | NOTHING { Expr.Nothing }
     | TRUE { Expr.Boolean true }