Skip to content

Commit 3eca3bf

Browse files
jepett0jepett0
authored
Merge ae3f2a4 into 47d509e 
2 parents 47d509e + ae3f2a4 commit 3eca3bf

File tree

6 files changed

+141
-22
lines changed

6 files changed

+141
-22
lines changed

‎ydb/core/kqp/compile_service/kqp_compile_actor.cpp‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -254,11 +254,10 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
254254
AsyncCompileResult->Continue().Apply(callback);
255255
}
256256

257-
void AddMessageToReplayLog(const TString& queryPlan) {
257+
void AddMessageToReplayLog(const TString& queryPlan, const TVector<NKikimrKqp::TKqpTableMetadataProto>& collectedSchemeData) {
258258
NJson::TJsonValue replayMessage(NJson::JSON_MAP);
259259

260260
NJson::TJsonValue tablesMeta(NJson::JSON_ARRAY);
261-
auto collectedSchemeData = Gateway->GetCollectedSchemeData();
262261
for (auto proto: collectedSchemeData) {
263262
tablesMeta.AppendValue(Base64Encode(proto.SerializeAsString()));
264263
}
@@ -372,10 +371,14 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
372371
PassAway();
373372
}
374373

375-
void FillCompileResult(std::unique_ptr<NKikimrKqp::TPreparedQuery> preparingQuery, NKikimrKqp::EQueryType queryType) {
374+
void FillCompileResult(std::unique_ptr<NKikimrKqp::TPreparedQuery> preparingQuery,
375+
NKikimrKqp::EQueryType queryType,
376+
const TVector<NKikimrKqp::TKqpTableMetadataProto>& collectedViewsMetadata = {}
377+
) {
376378
auto preparedQueryHolder = std::make_shared<TPreparedQueryHolder>(
377379
preparingQuery.release(), AppData()->FunctionRegistry);
378380
preparedQueryHolder->MutableLlvmSettings().Fill(Config, queryType);
381+
preparedQueryHolder->FillViews(collectedViewsMetadata);
379382
KqpCompileResult->PreparedQuery = preparedQueryHolder;
380383
KqpCompileResult->AllowCache = CanCacheQuery(KqpCompileResult->PreparedQuery->GetPhysicalQuery());
381384

@@ -403,19 +406,21 @@ class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> {
403406
Counters->ReportSqlVersion(DbCounters, *kqpResult.SqlVersion);
404407
}
405408

406-
if (status == Ydb::StatusIds::SUCCESS) {
407-
AddMessageToReplayLog(kqpResult.QueryPlan);
408-
}
409-
410409
ETableReadType maxReadType = ExtractMostHeavyReadType(kqpResult.QueryPlan);
411410

412411
auto queryType = QueryId.Settings.QueryType;
413412

414413
KqpCompileResult = TKqpCompileResult::Make(Uid, status, kqpResult.Issues(), maxReadType, std::move(QueryId));
415414

416415
if (status == Ydb::StatusIds::SUCCESS) {
416+
auto collectedSchemeData = Gateway->GetCollectedSchemeData();
417+
AddMessageToReplayLog(kqpResult.QueryPlan, collectedSchemeData);
418+
EraseIf(collectedSchemeData, [](const NKikimrKqp::TKqpTableMetadataProto& metadata) {
419+
return !metadata.HasKind() || static_cast<EKikimrTableKind>(metadata.GetKind()) != EKikimrTableKind::View;
420+
});
421+
417422
YQL_ENSURE(kqpResult.PreparingQuery);
418-
FillCompileResult(std::move(kqpResult.PreparingQuery), queryType);
423+
FillCompileResult(std::move(kqpResult.PreparingQuery), queryType, collectedSchemeData);
419424

420425
auto now = TInstant::Now();
421426
auto duration = now - StartTime;

‎ydb/core/kqp/query_data/kqp_prepared_query.cpp‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,22 @@ void TPreparedQueryHolder::FillTables(const google::protobuf::RepeatedPtrField<
267267
}
268268
}
269269
}
270+
271+
void TPreparedQueryHolder::FillViews(const TVector<NKikimrKqp::TKqpTableMetadataProto>& viewsMetadata) {
272+
for (const auto& view : viewsMetadata) {
273+
const auto& pathId = view.GetPathId();
274+
const auto schemaVersion = view.GetSchemaVersion();
275+
const auto& tableName = view.GetName();
276+
277+
NKqpProto::TKqpTableInfo tableInfo;
278+
tableInfo.MutableTableId()->SetOwnerId(pathId.GetOwnerId());
279+
tableInfo.MutableTableId()->SetTableId(pathId.GetTableId());
280+
tableInfo.SetSchemaVersion(schemaVersion);
281+
tableInfo.SetTableName(tableName);
282+
283+
QueryViews.emplace_back(std::move(tableInfo));
284+
}
285+
}
270286

271287
bool TPreparedQueryHolder::HasTempTables(TKqpTempTablesState::TConstPtr tempTablesState, bool withSessionId) const {
272288
if (!tempTablesState) {

‎ydb/core/kqp/query_data/kqp_prepared_query.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ class TPreparedQueryHolder {
137137
std::shared_ptr<const NKikimrKqp::TPreparedQuery> Proto;
138138
std::shared_ptr<TPreparedQueryAllocHolder> Alloc;
139139
TVector<TString> QueryTables;
140+
TVector<NKqpProto::TKqpTableInfo> QueryViews;
140141
std::vector<TKqpPhyTxHolder::TConstPtr> Transactions;
141142
TIntrusivePtr<TTableConstInfoMap> TableConstInfoById;
142143

@@ -180,6 +181,10 @@ class TPreparedQueryHolder {
180181
return QueryTables;
181182
}
182183

184+
const TVector<NKqpProto::TKqpTableInfo>& GetQueryViews() const {
185+
return QueryViews;
186+
}
187+
183188
const NKqpProto::TKqpPhyQuery& GetPhysicalQuery() const {
184189
return Proto->GetPhysicalQuery();
185190
}
@@ -192,6 +197,8 @@ class TPreparedQueryHolder {
192197

193198
void FillTables(const google::protobuf::RepeatedPtrField< ::NKqpProto::TKqpPhyStage>& stages);
194199

200+
void FillViews(const TVector<NKikimrKqp::TKqpTableMetadataProto>& viewsMetadata);
201+
195202
bool HasTempTables(TKqpTempTablesState::TConstPtr tempTablesState, bool withSessionId) const;
196203
};
197204

‎ydb/core/kqp/session_actor/kqp_query_state.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,16 @@ class TKqpQueryState : public TNonCopyable {
205205
TableVersions.emplace(tableId, table.GetVersion());
206206
}
207207
};
208+
auto addView = [&](const NKqpProto::TKqpTableInfo& view) {
209+
NKikimr::TTableId tableId(view.GetTableId().GetOwnerId(), view.GetTableId().GetTableId());
210+
auto it = TableVersions.find(tableId);
211+
if (it != TableVersions.end()) {
212+
Y_ENSURE(it->second == view.GetSchemaVersion());
213+
} else {
214+
TableVersions.emplace(tableId, view.GetSchemaVersion());
215+
}
216+
};
217+
208218
for (const auto& stage : phyTx.GetStages()) {
209219
for (const auto& tableOp : stage.GetTableOps()) {
210220
addTable(tableOp.GetTable());
@@ -232,6 +242,9 @@ class TKqpQueryState : public TNonCopyable {
232242
addTable(table.GetId());
233243
}
234244
}
245+
for (const auto& view : PreparedQuery->GetQueryViews()) {
246+
addView(view);
247+
}
235248
}
236249

237250
bool NeedCheckTableVersions() const {

‎ydb/core/kqp/ut/view/view_ut.cpp‎

Lines changed: 86 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
22
#include <ydb/library/yql/sql/sql.h>
33
#include <ydb/library/yql/utils/log/log.h>
4+
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
45

56
#include <util/folder/filelist.h>
67

78
#include <format>
89

910
using namespace NKikimr;
1011
using namespace NKikimr::NKqp;
12+
using namespace NYdb;
1113
using namespace NYdb::NTable;
1214

1315
namespace {
@@ -75,8 +77,37 @@ TDataQueryResult ExecuteDataModificationQuery(TSession& session,
7577
return result;
7678
}
7779

78-
TString GetYsonResults(TSession& session, const TString& query, const TExecDataQuerySettings& settings = {}) {
79-
return FormatResultSetYson(ExecuteDataModificationQuery(session, query, settings).GetResultSet(0));
80+
TValue GetSingleResult(const TDataQueryResult& rawResults) {
81+
auto resultSetParser = rawResults.GetResultSetParser(0);
82+
UNIT_ASSERT(resultSetParser.TryNextRow());
83+
return resultSetParser.GetValue(0);
84+
}
85+
86+
TValue GetSingleResult(TSession& session, const TString& query, const TExecDataQuerySettings& settings = {}) {
87+
return GetSingleResult(ExecuteDataModificationQuery(session, query, settings));
88+
}
89+
90+
TInstant GetTimestamp(const TValue& value) {
91+
return TValueParser(value).GetTimestamp();
92+
}
93+
94+
int GetInteger(const TValue& value) {
95+
return TValueParser(value).GetInt32();
96+
}
97+
98+
TMaybe<bool> GetFromCacheStat(const TQueryStats& stats) {
99+
const auto& proto = TProtoAccessor::GetProto(stats);
100+
if (!proto.Hascompilation()) {
101+
return Nothing();
102+
}
103+
return proto.Getcompilation().Getfrom_cache();
104+
}
105+
106+
void AssertFromCache(const TMaybe<TQueryStats>& stats, bool expectedValue) {
107+
UNIT_ASSERT(stats.Defined());
108+
const auto isFromCache = GetFromCacheStat(*stats);
109+
UNIT_ASSERT_C(isFromCache.Defined(), stats->ToString());
110+
UNIT_ASSERT_VALUES_EQUAL_C(*isFromCache, expectedValue, stats->ToString());
80111
}
81112

82113
void CompareResults(const TDataQueryResult& first, const TDataQueryResult& second) {
@@ -384,6 +415,53 @@ Y_UNIT_TEST_SUITE(TSelectFromViewTest) {
384415
ExecuteDataDefinitionQuery(session, ReadWholeFile(pathPrefix + "drop_view.sql"));
385416
}
386417
}
418+
419+
Y_UNIT_TEST(QueryCacheIsUpdated) {
420+
TKikimrRunner kikimr(TKikimrSettings().SetWithSampleTables(false));
421+
EnableViewsFeatureFlag(kikimr);
422+
auto session = kikimr.GetTableClient().CreateSession().GetValueSync().GetSession();
423+
424+
constexpr const char* viewName = "TheView";
425+
426+
const auto getCreationQuery = [&viewName](const char* innerQuery) -> TString {
427+
return std::format(R"(
428+
CREATE VIEW {} WITH (security_invoker = TRUE) AS {};
429+
)",
430+
viewName,
431+
innerQuery
432+
);
433+
};
434+
constexpr const char* firstInnerQuery = "SELECT 1";
435+
ExecuteDataDefinitionQuery(session, getCreationQuery(firstInnerQuery));
436+
437+
const TString selectFromViewQuery = std::format(R"(
438+
SELECT * FROM {};
439+
)",
440+
viewName
441+
);
442+
TExecDataQuerySettings queryExecutionSettings;
443+
queryExecutionSettings.KeepInQueryCache(true);
444+
queryExecutionSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
445+
ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
446+
// make sure the server side cache is working by calling the same query twice
447+
const auto cachedQueryRawResult = ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
448+
AssertFromCache(cachedQueryRawResult.GetStats(), true);
449+
UNIT_ASSERT_VALUES_EQUAL(GetInteger(GetSingleResult(cachedQueryRawResult)), 1);
450+
451+
// recreate the view with a different query inside
452+
ExecuteDataDefinitionQuery(session, std::format(R"(
453+
DROP VIEW {};
454+
)",
455+
viewName
456+
)
457+
);
458+
constexpr const char* secondInnerQuery = "SELECT 2";
459+
ExecuteDataDefinitionQuery(session, getCreationQuery(secondInnerQuery));
460+
461+
const auto secondCallRawResult = ExecuteDataModificationQuery(session, selectFromViewQuery, queryExecutionSettings);
462+
AssertFromCache(secondCallRawResult.GetStats(), false);
463+
UNIT_ASSERT_VALUES_EQUAL(GetInteger(GetSingleResult(secondCallRawResult)), 2);
464+
}
387465
}
388466

389467
Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
@@ -414,9 +492,9 @@ Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
414492
TExecDataQuerySettings queryExecutionSettings;
415493
queryExecutionSettings.KeepInQueryCache(true);
416494
const auto executeTwice = [&](const TString& query) {
417-
return TVector<TString>{
418-
GetYsonResults(session, query, queryExecutionSettings),
419-
GetYsonResults(session, query, queryExecutionSettings)
495+
return TVector<TInstant>{
496+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings)),
497+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings))
420498
};
421499
};
422500
const auto viewResults = executeTwice(selectFromViewQuery);
@@ -455,9 +533,9 @@ Y_UNIT_TEST_SUITE(TEvaluateExprInViewTest) {
455533
TExecDataQuerySettings queryExecutionSettings;
456534
queryExecutionSettings.KeepInQueryCache(true);
457535
const auto executeTwice = [&](const TString& query) {
458-
return TVector<TString>{
459-
GetYsonResults(session, query, queryExecutionSettings),
460-
GetYsonResults(session, query, queryExecutionSettings)
536+
return TVector<TInstant>{
537+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings)),
538+
GetTimestamp(GetSingleResult(session, query, queryExecutionSettings))
461539
};
462540
};
463541
const auto viewResults = executeTwice(selectFromViewQuery);

‎ydb/core/protos/kqp.proto‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -111,17 +111,17 @@ message TQueryRequest {
111111
}
112112

113113
message TKqpPathIdProto {
114-
optional uint32 OwnerId = 1;
115-
optional uint32 TableId = 2;
114+
optional uint64 OwnerId = 1;
115+
optional uint64 TableId = 2;
116116
}
117117

118118
message TIndexDescriptionProto {
119119
optional string Name = 1;
120120
optional uint32 Type = 2;
121121
optional uint32 State = 3;
122-
optional uint32 SchemaVersion = 4;
123-
optional uint32 LocalPathId = 5;
124-
optional uint32 PathOwnerId = 6;
122+
optional uint64 SchemaVersion = 4;
123+
optional uint64 LocalPathId = 5;
124+
optional uint64 PathOwnerId = 6;
125125
repeated string KeyColumns = 7;
126126
repeated string DataColumns = 8;
127127
};
@@ -160,7 +160,7 @@ message TKqpTableMetadataProto {
160160
optional string Name = 3;
161161
optional string SysView = 4;
162162
optional TKqpPathIdProto PathId = 5;
163-
optional uint32 SchemaVersion = 6;
163+
optional uint64 SchemaVersion = 6;
164164
optional uint32 Kind = 7;
165165
repeated TAttributeProto Attributes = 8;
166166
repeated TKqpColumnMetadataProto Columns = 9;

0 commit comments

Comments
 (0)