Painless: modify grammar to allow more statement delimiters (#29566)

This allows the grammar to determine when and what delimiters statements will use by
splitting up the statements into regular statements and delimited statements, those that do
not require a delimiter versus those that do. This allows consumers of the statements to
determine what delimiters the statements will use so that in certain cases semicolons are
not necessary like when there's a closing right bracket.

This change removes the need for semicolon insertion in the lexer, simplifying the existing
lexer quite a bit. It also ensures that there isn't a need to track semicolons being inserted
into places that aren't necessary such as array initializers.

Author: jdconrad

modules/lang-painless/src/main/antlr/PainlessParser.g4

@@ -22,7 +22,7 @@ parser grammar PainlessParser;
 options { tokenVocab=PainlessLexer; }
 
 source
-    : function* statement* EOF
+    : function* statement* dstatement? EOF
     ;
 
 function
@@ -33,23 +33,31 @@ parameters
     : LP ( decltype ID ( COMMA decltype ID )* )? RP
     ;
 
+statement
+    : rstatement
+    | dstatement SEMICOLON
+    ;
+
 // Note we use a predicate on the if/else case here to prevent the
 // "dangling-else" ambiguity by forcing the 'else' token to be consumed
 // as soon as one is found.  See (https://en.wikipedia.org/wiki/Dangling_else).
-statement
+rstatement
     : IF LP expression RP trailer ( ELSE trailer | { _input.LA(1) != ELSE }? )                 # if
     | WHILE LP expression RP ( trailer | empty )                                               # while
-    | DO block WHILE LP expression RP delimiter                                                # do
     | FOR LP initializer? SEMICOLON expression? SEMICOLON afterthought? RP ( trailer | empty ) # for
     | FOR LP decltype ID COLON expression RP trailer                                           # each
     | FOR LP ID IN expression RP trailer                                                       # ineach
-    | declaration delimiter                                                                    # decl
-    | CONTINUE delimiter                                                                       # continue
-    | BREAK delimiter                                                                          # break
-    | RETURN expression delimiter                                                              # return
     | TRY block trap+                                                                          # try
-    | THROW expression delimiter                                                               # throw
-    | expression delimiter                                                                     # expr
+    ;
+
+dstatement
+    : DO block WHILE LP expression RP # do
+    | declaration                     # decl
+    | CONTINUE                        # continue
+    | BREAK                           # break
+    | RETURN expression               # return
+    | THROW expression                # throw
+    | expression                      # expr
     ;
 
 trailer
@@ -58,7 +66,7 @@ trailer
     ;
 
 block
-    : LBRACK statement* RBRACK
+    : LBRACK statement* dstatement? RBRACK
     ;
 
 empty
@@ -90,11 +98,6 @@ trap
     : CATCH LP TYPE ID RP block
     ;
 
-delimiter
-    : SEMICOLON
-    | EOF
-    ;
-
 expression
     :               unary                                                 # single
     |               expression ( MUL | DIV | REM ) expression             # binary
@@ -169,8 +172,8 @@ braceaccess
     ;
 
 arrayinitializer
-    : NEW TYPE ( LBRACE expression RBRACE )+ ( postdot postfix* )?                                   # newstandardarray
-    | NEW TYPE LBRACE RBRACE LBRACK ( expression ( COMMA expression )* )? SEMICOLON? RBRACK postfix* # newinitializedarray
+    : NEW TYPE ( LBRACE expression RBRACE )+ ( postdot postfix* )?                        # newstandardarray
+    | NEW TYPE LBRACE RBRACE LBRACK ( expression ( COMMA expression )* )? RBRACK postfix* # newinitializedarray
     ;
 
 listinitializer
@@ -206,10 +209,8 @@ lamtype
     ;
 
 funcref
-    : TYPE REF ID      # classfuncref       // reference to a static or instance method,
-                                            // e.g. ArrayList::size or Integer::compare
-    | decltype REF NEW # constructorfuncref // reference to a constructor, e.g. ArrayList::new
-    | ID REF ID        # capturingfuncref   // reference to an instance method, e.g. object::toString
-                                            // currently limited to capture of a simple variable (id).
-    | THIS REF ID      # localfuncref       // reference to a local function, e.g. this::myfunc
+    : TYPE REF ID      # classfuncref
+    | decltype REF NEW # constructorfuncref
+    | ID REF ID        # capturingfuncref
+    | THIS REF ID      # localfuncref
     ;

modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java

@@ -44,36 +44,18 @@ final class EnhancedPainlessLexer extends PainlessLexer {
     private final String sourceName;
     private final Definition definition;
 
-    private Token stashedNext = null;
-    private Token previous = null;
+    private Token current = null;
 
     EnhancedPainlessLexer(CharStream charStream, String sourceName, Definition definition) {
         super(charStream);
         this.sourceName = sourceName;
         this.definition = definition;
     }
 
-    public Token getPreviousToken() {
-        return previous;
-    }
-
     @Override
     public Token nextToken() {
-        if (stashedNext != null) {
-            previous = stashedNext;
-            stashedNext = null;
-            return previous;
-        }
-        Token next = super.nextToken();
-        if (insertSemicolon(previous, next)) {
-            stashedNext = next;
-            previous = _factory.create(new Pair<TokenSource, CharStream>(this, _input), PainlessLexer.SEMICOLON, ";",
-                    Lexer.DEFAULT_TOKEN_CHANNEL, next.getStartIndex(), next.getStopIndex(), next.getLine(), next.getCharPositionInLine());
-            return previous;
-        } else {
-            previous = next;
-            return next;
-        }
+        current = super.nextToken();
+        return current;
     }
 
     @Override
@@ -101,7 +83,7 @@ protected boolean isSimpleType(String name) {
 
     @Override
     protected boolean slashIsRegex() {
-        Token lastToken = getPreviousToken();
+        Token lastToken = current;
         if (lastToken == null) {
             return true;
         }
@@ -120,18 +102,4 @@ protected boolean slashIsRegex() {
             return true;
         }
     }
-
-    private static boolean insertSemicolon(Token previous, Token next) {
-        if (previous == null || next.getType() != PainlessLexer.RBRACK) {
-            return false;
-        }
-        switch (previous.getType()) {
-        case PainlessLexer.RBRACK:     // };} would be weird!
-        case PainlessLexer.SEMICOLON:  // already have a semicolon, no need to add one
-        case PainlessLexer.LBRACK:     // empty blocks don't need a semicolon
-            return false;
-        default:
-            return true;
-        }
-    }
 }