Skip to content

Commit 4dd77d5

Browse files
vitstnvitstn
authored
Merge 7182fc5 into cba7316 
2 parents cba7316 + 7182fc5 commit 4dd77d5

File tree

10 files changed

+92
-14
lines changed

10 files changed

+92
-14
lines changed

‎ydb/library/yql/sql/settings/translation_settings.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ namespace NSQLTranslation {
6060
, V0ForceDisable(InTestEnvironment())
6161
, PGDisable(false)
6262
, WarnOnV0(true)
63+
, TestAntlr4(false)
6364
, V0WarnAsError(ISqlFeaturePolicy::MakeAlwaysDisallow())
6465
, DqDefaultAuto(ISqlFeaturePolicy::MakeAlwaysDisallow())
6566
, BlockDefaultAuto(ISqlFeaturePolicy::MakeAlwaysDisallow())

‎ydb/library/yql/sql/settings/translation_settings.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ namespace NSQLTranslation {
108108
bool V0ForceDisable;
109109
bool PGDisable;
110110
bool WarnOnV0;
111+
bool TestAntlr4;
111112
ISqlFeaturePolicy::TPtr V0WarnAsError;
112113
ISqlFeaturePolicy::TPtr DqDefaultAuto;
113114
ISqlFeaturePolicy::TPtr BlockDefaultAuto;

‎ydb/library/yql/sql/sql.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ namespace NSQLTranslation {
9696

9797
return NSQLTranslationV0::SqlAST(query, queryName, issues, maxErrors, settings.Arena);
9898
case 1:
99-
return NSQLTranslationV1::SqlAST(query, queryName, issues, maxErrors, parsedSettings.AnsiLexer, settings.Arena);
99+
return NSQLTranslationV1::SqlAST(query, queryName, issues, maxErrors, parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser, parsedSettings.TestAntlr4, settings.Arena);
100100
default:
101101
issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR,
102102
TStringBuilder() << "Unknown SQL syntax version: " << parsedSettings.SyntaxVersion));

‎ydb/library/yql/sql/v1/format/sql_format.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2872,7 +2872,7 @@ class TSqlFormatter : public NSQLFormat::ISqlFormatter {
28722872
}
28732873

28742874
if (mode == EFormatMode::Obfuscate) {
2875-
auto message = NSQLTranslationV1::SqlAST(query, "Query", issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser, parsedSettings.Arena);
2875+
auto message = NSQLTranslationV1::SqlAST(query, "Query", issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser, parsedSettings.TestAntlr4, parsedSettings.Arena);
28762876
if (!message) {
28772877
return false;
28782878
}
@@ -2935,7 +2935,7 @@ class TSqlFormatter : public NSQLFormat::ISqlFormatter {
29352935
}
29362936

29372937
NYql::TIssues parserIssues;
2938-
auto message = NSQLTranslationV1::SqlAST(currentQuery, "Query", parserIssues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser, parsedSettings.Arena);
2938+
auto message = NSQLTranslationV1::SqlAST(currentQuery, "Query", parserIssues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, parsedSettings.AnsiLexer, parsedSettings.Antlr4Parser, parsedSettings.TestAntlr4, parsedSettings.Arena);
29392939
if (!message) {
29402940
finalFormattedQuery << currentQuery;
29412941
if (!currentQuery.EndsWith("\n")) {

‎ydb/library/yql/sql/v1/proto_parser/proto_parser.cpp‎

Lines changed: 75 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414

1515
#include <ydb/library/yql/parser/proto_ast/gen/v1_proto_split/SQLv1Parser.pb.main.h>
1616

17+
#include <library/cpp/protobuf/util/simple_reflection.h>
18+
1719
#if defined(_tsan_enabled_)
1820
#include <util/system/mutex.h>
1921
#endif
@@ -29,18 +31,70 @@ namespace NSQLTranslationV1 {
2931

3032
using namespace NSQLv1Generated;
3133

32-
google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, TIssues& err, size_t maxErrors, bool ansiLexer, bool anlr4Parser, google::protobuf::Arena* arena) {
34+
void ValidateMessages(const google::protobuf::Message* msg1, const google::protobuf::Message* msg2) {
35+
YQL_ENSURE(!msg1 == !msg2);
36+
if (!msg1) {
37+
return;
38+
}
39+
40+
YQL_ENSURE(msg1->GetDescriptor() == msg2->GetDescriptor());
41+
const auto descr = msg1->GetDescriptor();
42+
if (descr == NSQLv1Generated::TToken::GetDescriptor()) {
43+
const auto& token1 = dynamic_cast<const NSQLv1Generated::TToken&>(*msg1);
44+
const auto& token2 = dynamic_cast<const NSQLv1Generated::TToken&>(*msg2);
45+
const bool isEof1 = token1.GetId() == Max<ui32>();
46+
const bool isEof2 = token2.GetId() == Max<ui32>();
47+
YQL_ENSURE(isEof1 == isEof2);
48+
YQL_ENSURE(token1.GetValue() == token2.GetValue());
49+
if (!isEof1) {
50+
YQL_ENSURE(token1.GetLine() == token2.GetLine());
51+
YQL_ENSURE(token1.GetColumn() == token2.GetColumn());
52+
}
53+
54+
return;
55+
}
56+
57+
for (int i = 0; i < descr->field_count(); ++i) {
58+
const NProtoBuf::FieldDescriptor* fd = descr->field(i);
59+
NProtoBuf::TConstField field1(*msg1, fd);
60+
NProtoBuf::TConstField field2(*msg2, fd);
61+
YQL_ENSURE(field1.IsMessage() == field2.IsMessage());
62+
if (field1.IsMessage()) {
63+
YQL_ENSURE(field1.Size() == field2.Size());
64+
for (size_t j = 0; j < field1.Size(); ++j) {
65+
ValidateMessages(field1.template Get<NProtoBuf::Message>(j), field2.template Get<NProtoBuf::Message>(j));
66+
}
67+
}
68+
}
69+
}
70+
71+
google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, TIssues& err,
72+
size_t maxErrors, bool ansiLexer, bool anlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) {
3373
YQL_ENSURE(arena);
3474
#if defined(_tsan_enabled_)
3575
TGuard<TMutex> grd(SanitizerSQLTranslationMutex);
3676
#endif
3777
NSQLTranslation::TErrorCollectorOverIssues collector(err, maxErrors, "");
3878
if (ansiLexer && !anlr4Parser) {
3979
NProtoAST::TProtoASTBuilder<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena);
40-
return builder.BuildAST(collector);
80+
auto res = builder.BuildAST(collector);
81+
if (testAntlr4) {
82+
NProtoAST::TProtoASTBuilder<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
83+
auto res2 = builder.BuildAST(collector);
84+
ValidateMessages(res, res2);
85+
}
86+
87+
return res;
4188
} else if (!ansiLexer && !anlr4Parser) {
4289
NProtoAST::TProtoASTBuilder<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena);
43-
return builder.BuildAST(collector);
90+
auto res = builder.BuildAST(collector);
91+
if (testAntlr4) {
92+
NProtoAST::TProtoASTBuilder<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
93+
auto res2 = builder.BuildAST(collector);
94+
ValidateMessages(res, res2);
95+
}
96+
97+
return res;
4498
} else if (ansiLexer && anlr4Parser) {
4599
NProtoAST::TProtoASTBuilder<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
46100
return builder.BuildAST(collector);
@@ -50,17 +104,32 @@ google::protobuf::Message* SqlAST(const TString& query, const TString& queryName
50104
}
51105
}
52106

53-
google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, bool ansiLexer, bool anlr4Parser, google::protobuf::Arena* arena) {
107+
google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err,
108+
bool ansiLexer, bool anlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) {
54109
YQL_ENSURE(arena);
55110
#if defined(_tsan_enabled_)
56111
TGuard<TMutex> grd(SanitizerSQLTranslationMutex);
57112
#endif
58113
if (ansiLexer && !anlr4Parser) {
59114
NProtoAST::TProtoASTBuilder<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena);
60-
return builder.BuildAST(err);
115+
auto res = builder.BuildAST(err);
116+
if (testAntlr4) {
117+
NProtoAST::TProtoASTBuilder<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
118+
auto res2 = builder.BuildAST(err);
119+
ValidateMessages(res, res2);
120+
}
121+
122+
return res;
61123
} else if (!ansiLexer && !anlr4Parser) {
62124
NProtoAST::TProtoASTBuilder<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena);
63-
return builder.BuildAST(err);
125+
auto res = builder.BuildAST(err);
126+
if (testAntlr4) {
127+
NProtoAST::TProtoASTBuilder<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
128+
auto res2 = builder.BuildAST(err);
129+
ValidateMessages(res, res2);
130+
}
131+
132+
return res;
64133
} else if (ansiLexer && anlr4Parser) {
65134
NProtoAST::TProtoASTBuilder<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
66135
return builder.BuildAST(err);

‎ydb/library/yql/sql/v1/proto_parser/proto_parser.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@ namespace NSQLTranslation {
1414

1515
namespace NSQLTranslationV1 {
1616

17-
google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, google::protobuf::Arena* arena = nullptr);
18-
google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, bool ansiLexer, bool antlr4Parser, google::protobuf::Arena* arena);
17+
google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
18+
NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena);
19+
google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
20+
NProtoAST::IErrorCollector& err, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena);
1921

2022
} // namespace NSQLTranslationV1

‎ydb/library/yql/sql/v1/sql.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTra
102102
TContext ctx(settings, hints, res.Issues);
103103
NSQLTranslation::TErrorCollectorOverIssues collector(res.Issues, settings.MaxErrors, settings.File);
104104

105-
google::protobuf::Message* ast(SqlAST(query, queryName, collector, settings.AnsiLexer, settings.Antlr4Parser, settings.Arena));
105+
google::protobuf::Message* ast(SqlAST(query, queryName, collector, settings.AnsiLexer, settings.Antlr4Parser, settings.TestAntlr4, settings.Arena));
106106
if (ast) {
107107
SqlASTToYqlImpl(res, *ast, ctx);
108108
} else {
@@ -195,7 +195,7 @@ TVector<NYql::TAstParseResult> SqlToAstStatements(const TString& query, const NS
195195
TContext ctx(settings, hints, issues);
196196
NSQLTranslation::TErrorCollectorOverIssues collector(issues, settings.MaxErrors, settings.File);
197197

198-
google::protobuf::Message* astProto(SqlAST(query, queryName, collector, settings.AnsiLexer, settings.Antlr4Parser, settings.Arena));
198+
google::protobuf::Message* astProto(SqlAST(query, queryName, collector, settings.AnsiLexer, settings.Antlr4Parser, settings.TestAntlr4, settings.Arena));
199199
if (astProto) {
200200
auto ast = static_cast<const TSQLv1ParserAST&>(*astProto);
201201
const auto& query = ast.GetRule_sql_query();

‎ydb/library/yql/tests/sql/sql2yql/test_sql2yql.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ def get_sql2yql_cmd(suite, case, case_file, out_dir, ansi_lexer, test_format, te
1313
cmd = [
1414
SQLRUN_PATH,
1515
case_file,
16-
'--syntax-version=1'
16+
'--syntax-version=1',
17+
'--test-antlr4',
1718
]
1819

1920
if ansi_lexer:

‎ydb/library/yql/tools/sql2yql/sql2yql.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ int BuildAST(int argc, char* argv[]) {
198198
opts.AddLongOption("assume-ydb-on-slash", "Assume YDB provider if cluster name starts with '/'").NoArgument();
199199
opts.AddLongOption("test-format", "compare formatted query's AST with the original query's AST (only syntaxVersion=1 is supported).").NoArgument();
200200
opts.AddLongOption("test-double-format", "check if formatting already formatted query produces the same result").NoArgument();
201+
opts.AddLongOption("test-antlr4", "check antlr4 parser").NoArgument();
201202
opts.AddLongOption("format-output", "Saves formatted query to it").RequiredArgument("format-output").StoreResult(&outFileNameFormat);
202203
opts.SetFreeArgDefaultTitle("query file");
203204
opts.AddHelpOption();
@@ -270,6 +271,7 @@ int BuildAST(int argc, char* argv[]) {
270271
settings.WarnOnV0 = false;
271272
settings.V0ForceDisable = false;
272273
settings.AssumeYdbOnClusterWithSlash = res.Has("assume-ydb-on-slash");
274+
settings.TestAntlr4 = res.Has("test-antlr4");
273275

274276
if (res.Has("lexer")) {
275277
NYql::TIssues issues;

‎ydb/library/yql/tools/yqlrun/yqlrun.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,7 @@ int Main(int argc, const char *argv[])
484484
opts.AddLongOption("pg-ext", "pg extensions config file").StoreResult(&pgExtConfig);
485485
opts.AddLongOption("with-final-issues", "Include some final messages (like statistic) in issues").NoArgument();
486486
opts.AddLongOption("validate-result-format", "Check that result-format can parse Result").NoArgument();
487+
opts.AddLongOption("test-antlr4", "check antlr4 parser").NoArgument();
487488

488489
opts.SetFreeArgsMax(0);
489490
TOptsParseResult res(&opts, argc, argv);
@@ -737,6 +738,7 @@ int Main(int argc, const char *argv[])
737738
settings.Flags = sqlFlags;
738739
settings.SyntaxVersion = syntaxVersion;
739740
settings.AnsiLexer = res.Has("ansi-lexer");
741+
settings.TestAntlr4 = res.Has("test-antlr4");
740742
settings.V0Behavior = NSQLTranslation::EV0Behavior::Report;
741743
settings.AssumeYdbOnClusterWithSlash = res.Has("assume-ydb-on-slash");
742744
if (res.Has("discover")) {

0 commit comments

Comments
 (0)