feat: optimize suggestion (#231)

* feat: optimize the strategy of finding the right range

* test: apply commentOtherLine util to all suggestion tests

* test: decomment suggestion test cases

* test: add suggestion test cases in multiple statements

* chore: improve comments

* test: update log info in test

Author: HaydenOrz

src/parser/common/basicParser.ts

@@ -277,55 +277,74 @@ export default abstract class BasicParser<
 
         /**
          * Split sql by statement.
-         * Try to collect candidates from the caret statement only.
+         * Try to collect candidates in as small a range as possible.
          */
         this.listen(splitListener, this._parseTree);
+        const statementCount = splitListener.statementsContext?.length;
+        const statementsContext = splitListener.statementsContext;
 
         // If there are multiple statements.
-        if (splitListener.statementsContext.length > 1) {
-            // find statement rule context where caretPosition is located.
-            const caretStatementContext = splitListener?.statementsContext.find((ctx) => {
-                return (
-                    caretTokenIndex <= ctx.stop?.tokenIndex &&
-                    caretTokenIndex >= ctx.start.tokenIndex
-                );
-            });
-
-            if (caretStatementContext) {
-                c3Context = caretStatementContext;
-            } else {
-                const lastStatementToken =
-                    splitListener.statementsContext[splitListener?.statementsContext.length - 1]
-                        .start;
+        if (statementCount > 1) {
+            /**
+             * Find a minimum valid range, reparse the fragment, and provide a new parse tree to C3.
+             * The boundaries of this range must be statements with no syntax errors.
+             * This can ensure the stable performance of the C3.
+             */
+            let startStatement: ParserRuleContext;
+            let stopStatement: ParserRuleContext;
+
+            for (let index = 0; index < statementCount; index++) {
+                const ctx = statementsContext[index];
+                const isCurrentCtxValid = !ctx.exception;
+                if (!isCurrentCtxValid) continue;
+
                 /**
-                 * If caretStatementContext is not found and it follows all statements.
-                 * Reparses part of the input following the penultimate statement.
-                 * And c3 will collect candidates in the new parseTreeContext.
+                 * Ensure that the statementContext before the left boundary
+                 * and the last statementContext on the right boundary are qualified SQL statements.
                  */
-                if (caretTokenIndex > lastStatementToken?.tokenIndex) {
-                    /**
-                     * Save offset of the tokenIndex in the partInput
-                     * compared to the tokenIndex in the whole input
-                     */
-                    tokenIndexOffset = lastStatementToken?.tokenIndex;
-                    // Correct caretTokenIndex
-                    caretTokenIndex = caretTokenIndex - tokenIndexOffset;
-
-                    const inputSlice = input.slice(lastStatementToken.startIndex);
-                    const lexer = this.createLexer(inputSlice);
-                    lexer.removeErrorListeners();
-
-                    const tokenStream = new CommonTokenStream(lexer);
-                    tokenStream.fill();
-                    const parser = this.createParserFromTokenStream(tokenStream);
-                    parser.removeErrorListeners();
-                    parser.buildParseTree = true;
-                    parser.errorHandler = new ErrorStrategy();
-
-                    sqlParserIns = parser;
-                    c3Context = parser.program();
+                const isPrevCtxValid = index === 0 || !statementsContext[index - 1]?.exception;
+                const isNextCtxValid =
+                    index === statementCount - 1 || !statementsContext[index + 1]?.exception;
+
+                if (ctx.stop.tokenIndex < caretTokenIndex && isPrevCtxValid) {
+                    startStatement = ctx;
+                }
+
+                if (!stopStatement && ctx.start.tokenIndex > caretTokenIndex && isNextCtxValid) {
+                    stopStatement = ctx;
+                    break;
                 }
             }
+
+            // A boundary consisting of the index of the input.
+            const startIndex = startStatement?.start?.startIndex ?? 0;
+            const stopIndex = stopStatement?.stop?.stopIndex ?? input.length - 1;
+
+            /**
+             * Save offset of the tokenIndex in the range of input
+             * compared to the tokenIndex in the whole input
+             */
+            tokenIndexOffset = startStatement?.start?.tokenIndex ?? 0;
+            caretTokenIndex = caretTokenIndex - tokenIndexOffset;
+
+            /**
+             * Reparse the input fragment，
+             * and c3 will collect candidates in the newly generated parseTree.
+             */
+            const inputSlice = input.slice(startIndex, stopIndex);
+
+            const lexer = this.createLexer(inputSlice);
+            lexer.removeErrorListeners();
+            const tokenStream = new CommonTokenStream(lexer);
+            tokenStream.fill();
+
+            const parser = this.createParserFromTokenStream(tokenStream);
+            parser.removeErrorListeners();
+            parser.buildParseTree = true;
+            parser.errorHandler = new ErrorStrategy();
+
+            sqlParserIns = parser;
+            c3Context = parser.program();
         }
 
         const core = new CodeCompletionCore(sqlParserIns);

test/parser/flinksql/suggestion/fixtures/multipleSql.sql

@@ -0,0 +1,31 @@
+SELECT * FROM  -- unfinished
+
+CREATE TEMPORARY VIEW IF NOT EXISTS v AS SELECT col1 FROM tbl;
+
+CREATE TEMPORARY TABLE client_errors (
+    log_time TIMESTAMP(3),
+    request_line STRING,
+    status_code STRING,
+    size INT
+) WITH (
+    'connector' = 'stream-x'
+);
+
+ALTER VIEW v1 RENAME TO v2;
+
+CREATE TABLE db. ; -- unfinished
+
+LOAD MODULE CORE;
+
+REMOVE JAR '<path_to_filename>.jar'
+
+INSERT INTO  VALUES (100, 99.9 / 10, 'abc', true, now ()); -- unfinished
+
+CREATE DATABASE IF NOT EXISTS dataApi COMMENT 'test create database' WITH ('key1' = 'value1', 'key2.a' = 'value2.a');
+
+DROP DATABASE IF EXISTS Orders RESTRICT;
+
+INSERT INTO country_page_view
+SELECT `user`,
+    cnt
+FROM db. ; -- unfinished

test/parser/flinksql/suggestion/fixtures/multipleStatement.sql

@@ -0,0 +1,69 @@
+import fs from 'fs';
+import path from 'path';
+import { CaretPosition, SyntaxContextType } from '../../../../src/parser/common/basic-parser-types';
+import FlinkSQL from '../../../../src/parser/flinksql';
+
+const syntaxSql = fs.readFileSync(
+    path.join(__dirname, 'fixtures', 'multipleStatement.sql'),
+    'utf-8'
+);
+
+describe('FlinkSQL Multiple Statements Syntax Suggestion', () => {
+    const parser = new FlinkSQL();
+
+    test('Select from table ', () => {
+        const pos: CaretPosition = {
+            lineNumber: 1,
+            column: 15,
+        };
+        const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === SyntaxContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual([]);
+    });
+
+    test('Create table ', () => {
+        const pos: CaretPosition = {
+            lineNumber: 16,
+            column: 17,
+        };
+        const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === SyntaxContextType.TABLE_CREATE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
+
+    test('Insert into table ', () => {
+        const pos: CaretPosition = {
+            lineNumber: 22,
+            column: 13,
+        };
+        const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === SyntaxContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual([]);
+    });
+
+    test('Insert into select from table ', () => {
+        const pos: CaretPosition = {
+            lineNumber: 31,
+            column: 9,
+        };
+        const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === SyntaxContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
+});

test/parser/flinksql/suggestion/multipleStatement.test.ts

@@ -8,7 +8,6 @@ const syntaxSql = fs.readFileSync(
     path.join(__dirname, 'fixtures', 'syntaxSuggestion.sql'),
     'utf-8'
 );
-const multipleSql = fs.readFileSync(path.join(__dirname, 'fixtures', 'multipleSql.sql'), 'utf-8');
 
 describe('Flink SQL Syntax Suggestion', () => {
     const parser = new FlinkSQL();
@@ -19,20 +18,6 @@ describe('Flink SQL Syntax Suggestion', () => {
         expect(parser.validate(syntaxSql).length).not.toBe(0);
     });
 
-    test('Multiple SQL use database', () => {
-        const pos: CaretPosition = {
-            lineNumber: 19,
-            column: 10,
-        };
-        const syntaxes = parser.getSuggestionAtCaretPosition(multipleSql, pos)?.syntax;
-        const suggestion = syntaxes?.find(
-            (syn) => syn.syntaxContextType === SyntaxContextType.DATABASE
-        );
-
-        expect(suggestion).not.toBeUndefined();
-        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['cat1', '.']);
-    });
-
     test('Drop catalog', () => {
         const pos: CaretPosition = {
             lineNumber: 1,

test/parser/flinksql/suggestion/syntaxSuggestion.test.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import { CaretPosition } from '../../../../src/parser/common/basic-parser-types';
 import FlinkSQL from '../../../../src/parser/flinksql';
+import { commentOtherLine } from '../../../helper';
 
 const tokenSql = fs.readFileSync(path.join(__dirname, 'fixtures', 'tokenSuggestion.sql'), 'utf-8');
 
@@ -13,7 +14,10 @@ describe('Flink SQL Token Suggestion', () => {
             lineNumber: 3,
             column: 5,
         };
-        const suggestion = parser.getSuggestionAtCaretPosition(tokenSql, pos)?.keywords;
+        const suggestion = parser.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, pos.lineNumber),
+            pos
+        )?.keywords;
 
         expect(suggestion).toEqual(['MODULES', 'CATALOG']);
     });
@@ -23,7 +27,10 @@ describe('Flink SQL Token Suggestion', () => {
             lineNumber: 5,
             column: 8,
         };
-        const suggestion = parser.getSuggestionAtCaretPosition(tokenSql, pos)?.keywords;
+        const suggestion = parser.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, pos.lineNumber),
+            pos
+        )?.keywords;
 
         expect(suggestion).toEqual([
             'CATALOG',
@@ -40,7 +47,10 @@ describe('Flink SQL Token Suggestion', () => {
             lineNumber: 7,
             column: 6,
         };
-        const suggestion = parser.getSuggestionAtCaretPosition(tokenSql, pos)?.keywords;
+        const suggestion = parser.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, pos.lineNumber),
+            pos
+        )?.keywords;
 
         expect(suggestion).toEqual([
             'MODULES',

test/parser/flinksql/suggestion/tokenSuggestion.test.ts

@@ -0,0 +1,21 @@
+SELECT * FROM  -- unfinished
+
+CREATE VIEW mydb.bro_view AS SELECT * FROM mydb.sale_tbl;
+
+CREATE TEMPORARY EXTERNAL TABLE list_bucket_multiple (col1 STRING, col2 INT, col3 STRING);
+
+ALTER VIEW myview1 SET TBLPROPERTIES ('author'='hayden','date'='2023-09-04')
+
+CREATE TABLE db. ; -- unfinished
+
+DROP CONNECTOR connector1;
+
+SET ROLE `admin`;
+
+INSERT INTO  VALUES (100, 99.9 / 10, 'abc', true, now ()); -- unfinished
+
+ALTER TABLE tbl1 RENAME TO tbl2;
+
+ALTER SCHEMA database_name SET OWNER USER `admin`;
+
+INSERT OVERWRITE LOCAL DIRECTORY '/path/to/output' SELECT col1, col2 FROM ; -- unfinished