Revert "24-4, VIEW: use parent context for query AST building"

ydb/library/yql/sql/sql.cpp

@@ -144,10 +144,6 @@ namespace NSQLTranslation {
     }
 
     NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings) {
-        return SqlASTToYql("", protoAst, hints, settings);
-    }
-
-    NYql::TAstParseResult SqlASTToYql(const TString& query, const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings) {
         NYql::TAstParseResult result;
         switch (settings.SyntaxVersion) {
             case 0:
@@ -165,7 +161,7 @@ namespace NSQLTranslation {
 
                 return NSQLTranslationV0::SqlASTToYql(protoAst, settings);
             case 1:
-                return NSQLTranslationV1::SqlASTToYql(query, protoAst, hints, settings);
+                return NSQLTranslationV1::SqlASTToYql(protoAst, hints, settings);
             default:
                 result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR,
                     TStringBuilder() << "Unknown SQL syntax version: " << settings.SyntaxVersion));

ydb/library/yql/sql/sql.h

@@ -21,11 +21,7 @@ namespace NSQLTranslation {
     google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NYql::TIssues& issues, size_t maxErrors,
         const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr);
     ILexer::TPtr SqlLexer(const TString& query, NYql::TIssues& issues, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr);
-
-    /*[[deprecated]] Use SqlASTToYql(query, protoAst, hints, settings)*/
     NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings);
-    NYql::TAstParseResult SqlASTToYql(const TString& query, const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings);
-
     TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const TTranslationSettings& settings,
         NYql::TWarningRules* warningRules = nullptr, ui16* actualSyntaxVersion = nullptr, TVector<NYql::TStmtParseInfo>* stmtParseInfo = nullptr);
 

ydb/library/yql/sql/v1/context.cpp

@@ -78,14 +78,12 @@ THashMap<TStringBuf, TPragmaMaybeField> CTX_PRAGMA_MAYBE_FIELDS = {
 
 TContext::TContext(const NSQLTranslation::TTranslationSettings& settings,
                    const NSQLTranslation::TSQLHints& hints,
-                   TIssues& issues,
-                   const TString& query)
+                   TIssues& issues)
     : ClusterMapping(settings.ClusterMapping)
     , PathPrefix(settings.PathPrefix)
     , ClusterPathPrefixes(settings.ClusterPathPrefixes)
     , SQLHints(hints)
     , Settings(settings)
-    , Query(query)
     , Pool(new TMemoryPool(4096))
     , Issues(issues)
     , IncrementMonCounterFunction(settings.IncrementCounter)

ydb/library/yql/sql/v1/context.h

@@ -87,8 +87,7 @@ namespace NSQLTranslationV1 {
     public:
         TContext(const NSQLTranslation::TTranslationSettings& settings,
                  const NSQLTranslation::TSQLHints& hints,
-                 NYql::TIssues& issues,
-                 const TString& query = {});
+                 NYql::TIssues& issues);
 
         virtual ~TContext();
 
@@ -233,7 +232,6 @@ namespace NSQLTranslationV1 {
         THashMap<TString, std::pair<TPosition, TNodePtr>> Variables;
         THashSet<TString> WeakVariables;
         NSQLTranslation::TTranslationSettings Settings;
-        const TString Query;
         std::unique_ptr<TMemoryPool> Pool;
         NYql::TIssues& Issues;
         TMap<TString, TNodePtr> UniversalAliases;
@@ -320,7 +318,6 @@ namespace NSQLTranslationV1 {
         ui64 ParallelModeCount = 0;
         bool CompactNamedExprs = false;
         bool ValidateUnusedExprs = false;
-        TVector<size_t> ForAllStatementsParts;
     };
 
     class TColumnRefScope {

ydb/library/yql/sql/v1/format/sql_format.cpp

@@ -35,6 +35,13 @@ TTokenIterator SkipWS(TTokenIterator curr, TTokenIterator end) {
     return curr;
 }
 
+TTokenIterator SkipWSOrComment(TTokenIterator curr, TTokenIterator end) {
+    while (curr != end && (curr->Name == "WS" || curr->Name == "COMMENT")) {
+        ++curr;
+    }
+    return curr;
+}
+
 bool Validate(const TParsedTokenList& query, const TParsedTokenList& formattedQuery) {
     auto in = query.begin();
     auto out = formattedQuery.begin();
@@ -66,6 +73,139 @@ bool Validate(const TParsedTokenList& query, const TParsedTokenList& formattedQu
     return in == inEnd && out == outEnd;
 }
 
+enum EParenType {
+    Open,
+    Close,
+    None
+};
+
+using TAdvanceCallback = std::function<EParenType(TTokenIterator& curr, TTokenIterator end)>;
+
+TTokenIterator SkipToNextBalanced(TTokenIterator begin, TTokenIterator end, const TAdvanceCallback& advance) {
+    i64 level = 0;
+    TTokenIterator curr = begin;
+    while (curr != end) {
+        switch (advance(curr, end)) {
+            case EParenType::Open: {
+                ++level;
+                break;
+            }
+            case EParenType::Close: {
+                --level;
+                if (level < 0) {
+                    return end;
+                } else if (level == 0) {
+                    return curr;
+                }
+                break;
+            }
+            case EParenType::None:
+                break;
+        }
+    }
+    return curr;
+}
+
+TTokenIterator GetNextStatementBegin(TTokenIterator begin, TTokenIterator end) {
+    TAdvanceCallback advanceLambdaBody = [](TTokenIterator& curr, TTokenIterator end) -> EParenType {
+        Y_UNUSED(end);
+        if (curr->Name == "LBRACE_CURLY") {
+            ++curr;
+            return EParenType::Open;
+        } else if (curr->Name == "RBRACE_CURLY") {
+            ++curr;
+            return EParenType::Close;
+        } else {
+            ++curr;
+            return EParenType::None;
+        }
+    };
+
+    TAdvanceCallback advanceAction = [](TTokenIterator& curr, TTokenIterator end) -> EParenType {
+        auto tmp = curr;
+        if (curr->Name == "DEFINE") {
+            ++curr;
+            curr = SkipWSOrComment(curr, end);
+            if (curr != end && (curr->Name == "ACTION" || curr->Name == "SUBQUERY")) {
+                ++curr;
+                return EParenType::Open;
+            }
+        } else if (curr->Name == "END") {
+            ++curr;
+            curr = SkipWSOrComment(curr, end);
+            if (curr != end && curr->Name == "DEFINE") {
+                ++curr;
+                return EParenType::Close;
+            }
+        }
+
+        curr = tmp;
+        ++curr;
+        return EParenType::None;
+    };
+
+    TAdvanceCallback advanceInlineAction = [](TTokenIterator& curr, TTokenIterator end) -> EParenType {
+        auto tmp = curr;
+        if (curr->Name == "DO") {
+            ++curr;
+            curr = SkipWSOrComment(curr, end);
+            if (curr != end && curr->Name == "BEGIN") {
+                ++curr;
+                return EParenType::Open;
+            }
+        } else if (curr->Name == "END") {
+            ++curr;
+            curr = SkipWSOrComment(curr, end);
+            if (curr != end && curr->Name == "DO") {
+                ++curr;
+                return EParenType::Close;
+            }
+        }
+
+        curr = tmp;
+        ++curr;
+        return EParenType::None;
+    };
+
+    TTokenIterator curr = begin;
+    while (curr != end) {
+        bool matched = false;
+        for (auto cb : {advanceLambdaBody, advanceAction, advanceInlineAction}) {
+            TTokenIterator tmp = curr;
+            if (cb(tmp, end) == EParenType::Open) {
+                curr = SkipToNextBalanced(curr, end, cb);
+                matched = true;
+                if (curr == end) {
+                    return curr;
+                }
+            }
+        }
+        if (matched) {
+            continue;
+        }
+        if (curr->Name == "SEMICOLON") {
+            ++curr;
+            break;
+        }
+        ++curr;
+    }
+
+    return curr;
+}
+
+void SplitByStatements(TTokenIterator begin, TTokenIterator end, TVector<TTokenIterator>& output) {
+    output.clear();
+    if (begin == end) {
+        return;
+    }
+    output.push_back(begin);
+    auto curr = begin;
+    while (curr != end) {
+        curr = GetNextStatementBegin(curr, end);
+        output.push_back(curr);
+    }
+}
+
 enum class EScope {
     Default,
     TypeName,
@@ -2709,13 +2849,40 @@ class TSqlFormatter : public NSQLFormat::ISqlFormatter {
         }
 
         auto lexer = NSQLTranslationV1::MakeLexer(parsedSettings.AnsiLexer);
-        TVector<TString> statements;
-        if (!NSQLTranslationV1::SplitQueryToStatements(query, lexer, statements, issues)) {
+        TParsedTokenList allTokens;
+        auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) {
+            if (token.Name != "EOF") {
+                allTokens.push_back(token);
+            }
+        };
+
+        if (!lexer->Tokenize(query, "Query", onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
             return false;
         }
 
+        TVector<TTokenIterator> statements;
+        SplitByStatements(allTokens.begin(), allTokens.end(), statements);
         TStringBuilder finalFormattedQuery;
-        for (const TString& currentQuery : statements) {
+        for (size_t i = 1; i < statements.size(); ++i) {
+            TStringBuilder currentQueryBuilder;
+            for (auto it = statements[i - 1]; it != statements[i]; ++it) {
+                currentQueryBuilder << it->Content;
+            }
+
+            TString currentQuery = currentQueryBuilder;
+            currentQuery = StripStringLeft(currentQuery);
+            bool isBlank = true;
+            for (auto c : currentQuery) {
+                if (c != ';') {
+                    isBlank = false;
+                    break;
+                }
+            };
+
+            if (isBlank) {
+                continue;
+            }
+
             TVector<NSQLTranslation::TParsedToken> comments;
             TParsedTokenList parsedTokens, stmtTokens;
             bool hasTrailingComments = false;