Merge a8416a5 into bc5b8c1

Author: nsofya

ydb/core/client/locks_ut.cpp

@@ -2,7 +2,7 @@
 #include <ydb/core/engine/mkql_engine_flat_impl.h>
 #include <ydb/core/testlib/test_client.h>
 #include <ydb/core/tx/tx_proxy/proxy.h>
-#include <ydb/core/tx/datashard/datashard_locks.h>
+#include <ydb/core/tx/locks/locks.h>
 #include <ydb/public/lib/deprecated/kicli/kicli.h>
 
 #include <library/cpp/testing/unittest/tests_data.h>

ydb/core/engine/minikql/minikql_engine_host.cpp

@@ -6,7 +6,7 @@
 #include <ydb/library/yql/minikql/computation/mkql_custom_list.h>
 #include <ydb/library/yql/minikql/mkql_string_util.h>
 #include <ydb/library/yql/parser/pg_wrapper/interface/codec.h>
-#include <ydb/core/tx/datashard/sys_tables.h>
+#include <ydb/core/tx/locks/sys_tables.h>
 
 #include <library/cpp/containers/stack_vector/stack_vec.h>
 

ydb/core/kqp/runtime/kqp_stream_lookup_worker.h

@@ -4,7 +4,7 @@
 #include <ydb/library/yql/minikql/mkql_node.h>
 #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
 #include <ydb/core/scheme/scheme_tabledefs.h>
-#include <ydb/core/tx/datashard/sys_tables.h>
+#include <ydb/core/tx/locks/sys_tables.h>
 #include <ydb/core/tx/datashard/datashard.h>
 
 namespace NKikimr {

ydb/core/kqp/ut/opt/kqp_ne_ut.cpp

@@ -2,6 +2,7 @@
 
 #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
 #include <ydb/core/kqp/runtime/kqp_read_actor.h>
+#include <ydb/core/tx/datashard/datashard_impl.h>
 
 namespace NKikimr::NKqp {
 

ydb/core/kqp/ut/scan/kqp_scan_ut.cpp

@@ -1,6 +1,7 @@
 #include <ydb/core/kqp/ut/common/kqp_ut_common.h>
 #include <ydb/core/kqp/counters/kqp_counters.h>
 #include <ydb/core/tx/datashard/datashard_failpoints.h>
+#include <ydb/core/tx/datashard/datashard_impl.h>
 #include <ydb/core/tx/scheme_cache/scheme_cache.h>
 
 #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>

ydb/core/kqp/ut/scan/kqp_split_ut.cpp

@@ -1,6 +1,7 @@
 #include <ydb/core/kqp/ut/common/kqp_ut_common.h>
 #include <ydb/core/kqp/counters/kqp_counters.h>
 #include <ydb/core/tx/scheme_cache/scheme_cache.h>
+#include <ydb/core/tx/datashard/datashard_impl.h>
 
 #include <ydb/core/base/tablet_pipecache.h>
 #include <ydb/core/kqp/runtime/kqp_read_actor.h>

ydb/core/sys_view/common/schema.h

@@ -3,7 +3,7 @@
 #include "path.h"
 
 #include <ydb/core/tablet_flat/flat_cxx_database.h>
-#include <ydb/core/tx/datashard/sys_tables.h>
+#include <ydb/core/tx/locks/sys_tables.h>
 
 namespace NKikimr {
 namespace NSysView {

ydb/core/tx/columnshard/columnshard__init.cpp

@@ -4,6 +4,7 @@
 #include "columnshard_schema.h"
 #include "hooks/abstract/abstract.h"
 #include <ydb/core/tx/columnshard/blobs_action/blob_manager_db.h>
+#include <ydb/core/tx/columnshard/transactions/locks_db.h>
 
 #include <ydb/core/tablet/tablet_exception.h>
 #include <ydb/core/tx/columnshard/operations/write.h>
@@ -36,6 +37,7 @@ void TTxInit::SetDefaults() {
     Self->LastWriteId = TWriteId{0};
     Self->LastPlannedStep = 0;
     Self->LastPlannedTxId = 0;
+    Self->LastCompletedTx = NOlap::TSnapshot::Zero();
     Self->OwnerPathId = 0;
     Self->OwnerPath.clear();
     Self->LongTxWrites.clear();
@@ -71,6 +73,14 @@ bool TTxInit::Precharge(TTransactionContext& txc) {
     ready = ready && Schema::GetSpecialValue(db, Schema::EValueIds::OwnerPathId, Self->OwnerPathId);
     ready = ready && Schema::GetSpecialValue(db, Schema::EValueIds::OwnerPath, Self->OwnerPath);
 
+    {
+        ui64 lastCompletedStep = 0;
+        ui64 lastCompletedTx = 0;
+        ready = ready && Schema::GetSpecialValue(db, Schema::EValueIds::LastCompletedStep, lastCompletedStep);
+        ready = ready && Schema::GetSpecialValue(db, Schema::EValueIds::LastCompletedTxId, lastCompletedTx);
+        Self->LastCompletedTx = NOlap::TSnapshot(lastCompletedStep, lastCompletedTx);
+    }
+
     if (!ready) {
         return false;
     }
@@ -177,6 +187,15 @@ bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx)
             }
         }
     }
+    {
+        TMemoryProfileGuard g("TTxInit/LocksDB");
+        if (txc.DB.GetScheme().GetTableInfo(Schema::Locks::TableId)) {
+            TColumnShardLocksDb locksDb(*Self, txc);
+            if (!Self->SysLocks.Load(locksDb)) {
+                return false;
+            }
+        }
+    }
 
     Self->UpdateInsertTableCounters();
     Self->UpdateIndexCounters();

ydb/core/tx/columnshard/columnshard__progress_tx.cpp

@@ -31,6 +31,12 @@ class TColumnShard::TTxProgressTx : public TTransactionBase<TColumnShard> {
         if (!!plannedItem) {
             ui64 step = plannedItem->PlanStep;
             ui64 txId = plannedItem->TxId;
+            LastCompletedTx = NOlap::TSnapshot(step, txId);
+            if (LastCompletedTx > Self->LastCompletedTx) {
+                NIceDb::TNiceDb db(txc.DB);
+                Schema::SaveSpecialValue(db, Schema::EValueIds::LastCompletedStep, LastCompletedTx->GetPlanStep());
+                Schema::SaveSpecialValue(db, Schema::EValueIds::LastCompletedTxId, LastCompletedTx->GetTxId());
+            }
 
             TxOperator = Self->ProgressTxController->GetVerifiedTxOperator(txId);
             AFL_VERIFY(TxOperator->Progress(*Self, NOlap::TSnapshot(step, txId), txc));
@@ -50,12 +56,16 @@ class TColumnShard::TTxProgressTx : public TTransactionBase<TColumnShard> {
         if (TxOperator) {
             TxOperator->Complete(*Self, ctx);
         }
+        if (LastCompletedTx) {
+            Self->LastCompletedTx = std::max(*LastCompletedTx, Self->LastCompletedTx);
+        }
         Self->SetupIndexation();
     }
 
 private:
     TTxController::ITransactionOperatior::TPtr TxOperator;
     const ui32 TabletTxNo;
+    std::optional<NOlap::TSnapshot> LastCompletedTx;
 };
 
 void TColumnShard::EnqueueProgressTx(const TActorContext& ctx) {

ydb/core/tx/columnshard/columnshard_impl.cpp

@@ -93,6 +93,7 @@ TColumnShard::TColumnShard(TTabletStorageInfo* info, const TActorId& tablet)
     , ScanCounters("Scan")
     , WritesMonitor(*this)
     , NormalizerController(StoragesManager, SubscribeCounters)
+    , SysLocks(this)
 {
     TabletCountersPtr.reset(new TProtobufTabletCounters<
         ESimpleCounters_descriptor,