default value for columns (#6101)

Author: ivanmorozov333

ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp

@@ -10,6 +10,8 @@ TConclusionStatus TAlterColumnOperation::DoDeserialize(NYql::TObjectSettingsImpl
         }
         ColumnName = *fValue;
     }
+    DefaultValue = features.Extract("DEFAULT_VALUE");
+
     StorageId = features.Extract("STORAGE_ID");
     if (StorageId && !*StorageId) {
         return TConclusionStatus::Fail("STORAGE_ID cannot be empty string");
@@ -39,6 +41,9 @@ void TAlterColumnOperation::DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTable
         Serializer.SerializeToProto(*column->MutableSerializer());
     }
     *column->MutableDictionaryEncoding() = DictionaryEncodingDiff.SerializeToProto();
+    if (DefaultValue) {
+        column->SetDefaultValue(*DefaultValue);
+    }
 }
 
 }

ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h

@@ -17,6 +17,7 @@ class TAlterColumnOperation : public ITableStoreOperation {
 
     NArrow::NSerialization::TSerializerContainer Serializer;
     NArrow::NDictionary::TEncodingDiff DictionaryEncodingDiff;
+    std::optional<TString> DefaultValue;
 public:
     TConclusionStatus DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) override;
 

ydb/core/kqp/ut/olap/write_ut.cpp

@@ -1,5 +1,8 @@
 #include "helpers/local.h"
 #include "helpers/writer.h"
+#include "helpers/typed_local.h"
+#include "helpers/query_executor.h"
+#include "helpers/get_value.h"
 
 #include <library/cpp/testing/unittest/registar.h>
 #include <ydb/core/tx/columnshard/hooks/testing/controller.h>
@@ -105,7 +108,27 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
             ("writes", writesCountStart)("count", Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->GetWritesCount());
         AFL_VERIFY(deletesCountStart == Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->GetDeletesCount())
             ("deletes", deletesCountStart)("count", Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->GetDeletesCount());
+    }
+
+    Y_UNIT_TEST(DefaultValues) {
+        auto settings = TKikimrSettings().SetWithSampleTables(false);
+        TKikimrRunner kikimr(settings);
+        Tests::NCommon::TLoggerInit(kikimr).Initialize();
+        TTypedLocalHelper helper("Utf8", kikimr);
+        helper.CreateTestOlapTable();
+        helper.ExecuteSchemeQuery("ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=ALTER_COLUMN, NAME=field, `ENCODING.DICTIONARY.ENABLED`=`true`, `DEFAULT_VALUE`=`abcde`);");
+        helper.FillPKOnly(0, 800000);
 
+        auto selectQuery = TString(R"(
+                SELECT
+                    count(*) as count,
+                FROM `/Root/olapStore/olapTable`
+                WHERE field = 'abcde'
+            )");
+
+        auto tableClient = kikimr.GetTableClient();
+        auto rows = ExecuteScanQuery(tableClient, selectQuery);
+        UNIT_ASSERT_VALUES_EQUAL(GetUint64(rows[0].at("count")), 800000);
     }
 
 }

ydb/core/protos/flat_scheme_op.proto

@@ -18,6 +18,7 @@ import "ydb/library/actors/protos/actors.proto";
 import "ydb/library/mkql_proto/protos/minikql.proto";
 import "ydb/core/protos/index_builder.proto";
 import "ydb/core/tx/columnshard/engines/scheme/statistics/protos/data.proto";
+import "ydb/core/tx/columnshard/engines/scheme/defaults/protos/data.proto";
 import "ydb/core/tx/columnshard/common/protos/snapshot.proto";
 
 import "google/protobuf/empty.proto";
@@ -421,6 +422,7 @@ message TOlapColumnDiff {
     optional TDictionaryEncodingSettings DictionaryEncoding = 4;
     optional TOlapColumn.TSerializer Serializer = 5;
     optional string StorageId = 6;
+    optional string DefaultValue = 7;
 }
 
 message TOlapColumnDescription {
@@ -439,6 +441,7 @@ message TOlapColumnDescription {
     optional TDictionaryEncodingSettings DictionaryEncoding = 9;
     optional TOlapColumn.TSerializer Serializer = 10;
     optional string StorageId = 11;
+    optional NKikimrColumnShardColumnDefaults.TColumnDefault DefaultValue = 12;
 }
 
 message TRequestedBloomFilter {

ydb/core/protos/ya.make

@@ -166,6 +166,7 @@ PEERDIR(
     ydb/library/services
     ydb/library/ydb_issue/proto
     ydb/core/tx/columnshard/engines/scheme/statistics/protos
+    ydb/core/tx/columnshard/engines/scheme/defaults/protos
     ydb/core/tx/columnshard/engines/protos
     ydb/core/formats/arrow/protos
     ydb/core/tx/columnshard/common/protos

ydb/core/tx/columnshard/blobs_reader/actor.h

@@ -22,7 +22,7 @@ class TActor: public TActorBootstrapped<TActor> {
     void Bootstrap();
 
     STFUNC(StateWait) {
-        TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD);
+        TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("event_type", ev->GetTypeName());
         switch (ev->GetTypeRewrite()) {
             hFunc(NBlobCache::TEvBlobCache::TEvReadBlobRangeResult, Handle);
             default:

ydb/core/tx/columnshard/engines/changes/general_compaction.cpp

@@ -183,7 +183,7 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc
                 } else {
                     AFL_VERIFY(dataSchema->IsSpecialColumnId(columnId));
                 }
-                chunks.emplace_back(std::make_shared<NChunks::TDefaultChunkPreparation>(columnId, p.GetPortionInfo().GetRecordsCount(), resultField, resultSchema->GetDefaultReadValueVerified(columnId), resultSchema->GetColumnSaver(columnId)));
+                chunks.emplace_back(std::make_shared<NChunks::TDefaultChunkPreparation>(columnId, p.GetPortionInfo().GetRecordsCount(), resultField, resultSchema->GetDefaultValueVerified(columnId), resultSchema->GetColumnSaver(columnId)));
                 records = { nullptr };
             }
             AFL_VERIFY(!!loader);

ydb/core/tx/columnshard/engines/portions/portion_info.cpp

@@ -713,21 +713,23 @@ std::shared_ptr<arrow::ChunkedArray> TPortionInfo::TPreparedColumn::Assemble() c
 }
 
 TDeserializeChunkedArray::TChunk TPortionInfo::TAssembleBlobInfo::BuildDeserializeChunk(const std::shared_ptr<TColumnLoader>& loader) const {
-    if (NullRowsCount) {
+    if (DefaultRowsCount) {
         Y_ABORT_UNLESS(!Data);
-        auto emptyBatch = NArrow::MakeEmptyBatch(loader->GetExpectedSchema(), NullRowsCount);
-        AFL_VERIFY(emptyBatch->num_columns() == 1);
-        return TDeserializeChunkedArray::TChunk(emptyBatch->column(0));
+        AFL_VERIFY(loader->GetExpectedSchema()->num_fields() == 1);
+        auto col = NArrow::TThreadSimpleArraysCache::Get(loader->GetExpectedSchema()->field(0)->type(), DefaultValue, DefaultRowsCount);
+        return TDeserializeChunkedArray::TChunk(col);
     } else {
         AFL_VERIFY(ExpectedRowsCount);
         return TDeserializeChunkedArray::TChunk(*ExpectedRowsCount, Data);
     }
 }
 
 std::shared_ptr<arrow::RecordBatch> TPortionInfo::TAssembleBlobInfo::BuildRecordBatch(const TColumnLoader& loader) const {
-    if (NullRowsCount) {
+    if (DefaultRowsCount) {
         Y_ABORT_UNLESS(!Data);
-        return NArrow::MakeEmptyBatch(loader.GetExpectedSchema(), NullRowsCount);
+        AFL_VERIFY(loader.GetExpectedSchema()->num_fields() == 1);
+        return arrow::RecordBatch::Make(loader.GetExpectedSchema(), DefaultRowsCount,
+            { NArrow::TThreadSimpleArraysCache::Get(loader.GetExpectedSchema()->field(0)->type(), DefaultValue, DefaultRowsCount) });
     } else {
         auto result = loader.Apply(Data);
         if (!result.ok()) {

ydb/core/tx/columnshard/engines/portions/portion_info.h

@@ -613,7 +613,8 @@ class TPortionInfo {
     class TAssembleBlobInfo {
     private:
         YDB_READONLY_DEF(std::optional<ui32>, ExpectedRowsCount);
-        ui32 NullRowsCount = 0;
+        ui32 DefaultRowsCount = 0;
+        std::shared_ptr<arrow::Scalar> DefaultValue;
         TString Data;
     public:
         ui32 GetExpectedRowsCountVerified() const {
@@ -625,34 +626,36 @@ class TPortionInfo {
             AFL_VERIFY(!ExpectedRowsCount);
             ExpectedRowsCount = expectedRowsCount;
             if (!Data) {
-                AFL_VERIFY(*ExpectedRowsCount == NullRowsCount);
+                AFL_VERIFY(*ExpectedRowsCount == DefaultRowsCount);
             }
         }
 
-        TAssembleBlobInfo(const ui32 rowsCount)
-            : NullRowsCount(rowsCount) {
-            AFL_VERIFY(NullRowsCount);
+        TAssembleBlobInfo(const ui32 rowsCount, const std::shared_ptr<arrow::Scalar>& defValue)
+            : DefaultRowsCount(rowsCount)
+            , DefaultValue(defValue)
+        {
+            AFL_VERIFY(DefaultRowsCount);
         }
 
         TAssembleBlobInfo(const TString& data)
             : Data(data) {
             AFL_VERIFY(!!Data);
         }
 
-        ui32 GetNullRowsCount() const noexcept {
-            return NullRowsCount;
+        ui32 GetDefaultRowsCount() const noexcept {
+            return DefaultRowsCount;
         }
 
         const TString& GetData() const noexcept {
             return Data;
         }
 
         bool IsBlob() const {
-            return !NullRowsCount && !!Data;
+            return !DefaultRowsCount && !!Data;
         }
 
-        bool IsNull() const {
-            return NullRowsCount && !Data;
+        bool IsDefault() const {
+            return DefaultRowsCount && !Data;
         }
 
         std::shared_ptr<arrow::RecordBatch> BuildRecordBatch(const TColumnLoader& loader) const;
@@ -788,7 +791,7 @@ class TPortionInfo {
 
         TPreparedColumn Compile() {
             if (BlobsInfo.empty()) {
-                BlobsInfo.emplace_back(TAssembleBlobInfo(NumRows));
+                BlobsInfo.emplace_back(TAssembleBlobInfo(NumRows, DataLoader->GetDefaultValue()));
                 return TPreparedColumn(std::move(BlobsInfo), ResultLoader);
             } else {
                 AFL_VERIFY(NumRowsByChunks == NumRows)("by_chunks", NumRowsByChunks)("expected", NumRows);

ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.cpp

@@ -10,8 +10,8 @@ void TFetchedData::SyncTableColumns(const std::vector<std::shared_ptr<arrow::Fie
         if (Table->GetSchema()->GetFieldByName(i->name())) {
             continue;
         }
-        Table->AddField(i, std::make_shared<NArrow::NAccessor::TTrivialArray>(NArrow::TThreadSimpleArraysCache::GetNull(i->type(), Table->num_rows())))
-            .Validate();
+        Table->AddField(i, std::make_shared<NArrow::NAccessor::TTrivialArray>(
+            NArrow::TThreadSimpleArraysCache::GetNull(i->type(), Table->num_rows()))).Validate();
     }
 }