Implemented limit on olap table columnt count (#8742)

aavdonkin · web-flow · commit 4efd77cd2ad1 · 2024-09-09T21:48:46.000+03:00
diff --git a/ydb/core/protos/flat_tx_scheme.proto b/ydb/core/protos/flat_tx_scheme.proto
@@ -190,6 +190,8 @@ message TSchemeLimits {
 
     optional uint64 MaxExports = 16;
     optional uint64 MaxImports = 17;
+
+    optional uint64 MaxColumnTableColumns = 18;
 }
 
 message TEvInitTenantSchemeShard {
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp
@@ -36,6 +36,21 @@ NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitializeImpl(const TUpda
             return TConclusionStatus::Fail("schema update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE");
         }
     }
+
+    const TString& parentPathStr = context.GetModification()->GetWorkingDir();
+    if (parentPathStr) { // Not empty only if called from Propose, not from ProgressState
+        NSchemeShard::TPath parentPath = NSchemeShard::TPath::Resolve(parentPathStr, context.GetSSOperationContext()->SS);
+        auto domainInfo = parentPath.DomainInfo();
+        const TSchemeLimits& limits = domainInfo->GetSchemeLimits();
+        if (targetSchema.GetColumns().GetColumns().size() > limits.MaxColumnTableColumns) {
+            TString errStr = TStringBuilder()
+                << "Too many columns"
+                << ": new: " << targetSchema.GetColumns().GetColumns().size()
+                << ". Limit: " << limits.MaxColumnTableColumns;
+            return TConclusionStatus::Fail(errStr);
+        }
+    }
+
     auto description = originalTable.GetTableInfoVerified().Description;
     targetSchema.Serialize(*description.MutableSchema());
     auto ttl = originalTable.GetTableTTLOptional() ? *originalTable.GetTableTTLOptional() : TOlapTTL();
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter_store.cpp b/ydb/core/tx/schemeshard/olap/operations/alter_store.cpp
@@ -2,6 +2,8 @@
 #include <ydb/core/tx/schemeshard/schemeshard__operation_common.h>
 #include <ydb/core/tx/schemeshard/schemeshard_impl.h>
 
+#include "checks.h"
+
 namespace {
 
 using namespace NKikimr;
@@ -458,7 +460,8 @@ class TAlterOlapStore: public TSubOperation {
             return result;
         }
 
-        TPath path = TPath::Resolve(parentPathStr, context.SS).Dive(name);
+        TPath parentPath = TPath::Resolve(parentPathStr, context.SS);
+        TPath path = parentPath.Dive(name);
         {
             TPath::TChecker checks = path.Check();
             checks
@@ -504,6 +507,15 @@ class TAlterOlapStore: public TSubOperation {
         if (!alterData) {
             return result;
         }
+
+        auto domainInfo = parentPath.DomainInfo();
+        const TSchemeLimits& limits = domainInfo->GetSchemeLimits();
+
+        if (!NKikimr::NSchemeShard::NOlap::CheckLimits(limits, alterData, errStr)) {
+            result->SetError(NKikimrScheme::StatusSchemeError, errStr);
+            return result;
+        }
+
         storeInfo->AlterData = alterData;
 
         NIceDb::TNiceDb db(context.GetDB());
diff --git a/ydb/core/tx/schemeshard/olap/operations/checks.h b/ydb/core/tx/schemeshard/olap/operations/checks.h
@@ -0,0 +1,19 @@
+#pragma once
+
+namespace NKikimr::NSchemeShard::NOlap {
+    inline bool CheckLimits(const TSchemeLimits& limits, TOlapStoreInfo::TPtr alterData, TString& errStr) {
+        for (auto& [_, preset]: alterData->SchemaPresets) {
+            ui64 columnCount = preset.GetColumns().GetColumns().size();
+            if (columnCount > limits.MaxColumnTableColumns) {
+                errStr = TStringBuilder()
+                    << "Too many columns"
+                    << ". new: " << columnCount
+                    << ". Limit: " << limits.MaxColumnTableColumns;
+                return false;
+            }
+        }
+        return true;
+    }
+}
+
+
diff --git a/ydb/core/tx/schemeshard/olap/operations/create_store.cpp b/ydb/core/tx/schemeshard/olap/operations/create_store.cpp
@@ -7,6 +7,8 @@
 #include <ydb/core/tx/columnshard/columnshard.h>
 #include <ydb/core/mind/hive/hive.h>
 
+#include "checks.h"
+
 using namespace NKikimr;
 using namespace NKikimr::NSchemeShard;
 
@@ -394,12 +396,20 @@ class TCreateOlapStore: public TSubOperation {
             return result;
         }
 
+        auto domainInfo = parentPath.DomainInfo();
+        const TSchemeLimits& limits = domainInfo->GetSchemeLimits();
+
         TProposeErrorCollector errors(*result);
         TOlapStoreInfo::TPtr storeInfo = std::make_shared<TOlapStoreInfo>();
         if (!storeInfo->ParseFromRequest(createDescription, errors)) {
             return result;
         }
 
+        if (!NKikimr::NSchemeShard::NOlap::CheckLimits(limits, storeInfo, errStr)) {
+            result->SetError(NKikimrScheme::StatusSchemeError, errStr);
+            return result;
+        }
+
         // Construct channels bindings for columnshards
         TChannelsBindings channelsBindings;
         if (!context.SS->GetOlapChannelsBindings(dstPath.GetPathIdForDomain(), storeInfo->GetStorageConfig(), channelsBindings, errStr)) {
diff --git a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp
@@ -681,11 +681,23 @@ class TCreateColumnTable: public TSubOperation {
         TProposeErrorCollector errors(*result);
         TColumnTableInfo::TPtr tableInfo;
         bool needUpdateObject = false;
+        auto domainInfo = parentPath.DomainInfo();
+        const TSchemeLimits& limits = domainInfo->GetSchemeLimits();
+
         if (storeInfo) {
             TOlapPresetConstructor tableConstructor(*storeInfo);
             tableInfo = tableConstructor.BuildTableInfo(createDescription, context, errors);
             needUpdateObject = tableConstructor.GetNeedUpdateObject();
         } else {
+            ui64 columnCount = createDescription.schema().columns().size();
+            if (columnCount > limits.MaxColumnTableColumns) {
+                TString errStr = TStringBuilder()
+                    << "Too many columns"
+                    << ". new: " << columnCount
+                    << ". Limit: " << limits.MaxColumnTableColumns;
+                result->SetError(NKikimrScheme::StatusSchemeError, errStr);
+                return result;
+            }
             TOlapTableConstructor tableConstructor;
             tableInfo = tableConstructor.BuildTableInfo(createDescription, context, errors);
         }
diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp
@@ -1243,6 +1243,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
             .MaxPathElementLength = rowSet.template GetValueOrDefault<Schema::SubDomains::PathElementLength>(defaults.MaxPathElementLength),
             .ExtraPathSymbolsAllowed = rowSet.template GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(defaults.ExtraPathSymbolsAllowed),
             .MaxTableColumns = rowSet.template GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(defaults.MaxTableColumns),
+            .MaxColumnTableColumns = rowSet.template GetValueOrDefault<Schema::SubDomains::ColumnTableColumnsLimit>(defaults.MaxColumnTableColumns),
             .MaxTableColumnNameLength = rowSet.template GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(defaults.MaxTableColumnNameLength),
             .MaxTableKeyColumns = rowSet.template GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(defaults.MaxTableKeyColumns),
             .MaxTableIndices = rowSet.template GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(defaults.MaxTableIndices),
diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h
@@ -768,6 +768,7 @@ struct Schema : NIceDb::Schema {
         struct ImportsLimit : Column<29, NScheme::NTypeIds::Uint64> {};
         struct AuditSettings : Column<30, NScheme::NTypeIds::String> {};
         struct ServerlessComputeResourcesMode : Column<31, NScheme::NTypeIds::Uint32> { using Type = EServerlessComputeResourcesMode; };
+        struct ColumnTableColumnsLimit : Column<32, NScheme::NTypeIds::Uint64> {};
 
         using TKey = TableKey<PathId>;
         using TColumns = TableColumns<
@@ -801,7 +802,8 @@ struct Schema : NIceDb::Schema {
             ExportsLimit,
             ImportsLimit,
             AuditSettings,
-            ServerlessComputeResourcesMode
+            ServerlessComputeResourcesMode,
+            ColumnTableColumnsLimit
         >;
     };
 
diff --git a/ydb/core/tx/schemeshard/schemeshard_types.cpp b/ydb/core/tx/schemeshard/schemeshard_types.cpp
@@ -20,6 +20,9 @@ TSchemeLimits TSchemeLimits::FromProto(const NKikimrScheme::TSchemeLimits& proto
     if (proto.HasMaxTableColumns()) {
         result.MaxTableColumns = proto.GetMaxTableColumns();
     }
+    if (proto.HasMaxColumnTableColumns()) {
+        result.MaxColumnTableColumns = proto.GetMaxColumnTableColumns();
+    }
     if (proto.HasMaxTableColumnNameLength()) {
         result.MaxTableColumnNameLength = proto.GetMaxTableColumnNameLength();
     }
@@ -69,6 +72,7 @@ NKikimrScheme::TSchemeLimits TSchemeLimits::AsProto() const {
     result.SetMaxAclBytesSize(MaxAclBytesSize);
 
     result.SetMaxTableColumns(MaxTableColumns);
+    result.SetMaxColumnTableColumns(MaxColumnTableColumns);
     result.SetMaxTableColumnNameLength(MaxTableColumnNameLength);
     result.SetMaxTableKeyColumns(MaxTableKeyColumns);
     result.SetMaxTableIndices(MaxTableIndices);
diff --git a/ydb/core/tx/schemeshard/schemeshard_types.h b/ydb/core/tx/schemeshard/schemeshard_types.h
@@ -39,6 +39,7 @@ struct TSchemeLimits {
 
     // table
     ui64 MaxTableColumns = 200;
+    ui64 MaxColumnTableColumns = 10000;
     ui64 MaxTableColumnNameLength = 255;
     ui64 MaxTableKeyColumns = 20;
     ui64 MaxTableIndices = 20;
diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp
@@ -1506,6 +1506,7 @@ namespace NSchemeShardUT_Private {
                                         (let child '('ChildrenLimit (Uint64 '%lu)))
                                         (let acl '('AclByteSizeLimit (Uint64 '%lu)))
                                         (let columns '('TableColumnsLimit (Uint64 '%lu)))
+                                        (let columnColumns '('ColumnTableColumnsLimit (Uint64 '%lu)))
                                         (let colName '('TableColumnNameLengthLimit (Uint64 '%lu)))
                                         (let keyCols '('TableKeyColumnsLimit (Uint64 '%lu)))
                                         (let indices '('TableIndicesLimit (Uint64 '%lu)))
@@ -1518,11 +1519,11 @@ namespace NSchemeShardUT_Private {
                                         (let pqPartitions '('PQPartitionsLimit (Uint64 '%lu)))
                                         (let exports '('ExportsLimit (Uint64 '%lu)))
                                         (let imports '('ImportsLimit (Uint64 '%lu)))
-                                        (let ret (AsList (UpdateRow 'SubDomains key '(depth paths child acl columns colName keyCols indices streams shards pathShards consCopy maxPathLength extraSymbols pqPartitions exports imports))))
+                                        (let ret (AsList (UpdateRow 'SubDomains key '(depth paths child acl columns columnColumns colName keyCols indices streams shards pathShards consCopy maxPathLength extraSymbols pqPartitions exports imports))))
                                         (return ret)
                                     )
                                  )", domainId, limits.MaxDepth, limits.MaxPaths, limits.MaxChildrenInDir, limits.MaxAclBytesSize,
-                               limits.MaxTableColumns, limits.MaxTableColumnNameLength, limits.MaxTableKeyColumns,
+                               limits.MaxTableColumns, limits.MaxColumnTableColumns, limits.MaxTableColumnNameLength, limits.MaxTableKeyColumns,
                                limits.MaxTableIndices, limits.MaxTableCdcStreams,
                                limits.MaxShards, limits.MaxShardsInPath, limits.MaxConsistentCopyTargets,
                                limits.MaxPathElementLength, escapedStr.c_str(), limits.MaxPQPartitions,
diff --git a/ydb/core/tx/schemeshard/ut_subdomain/ut_subdomain.cpp b/ydb/core/tx/schemeshard/ut_subdomain/ut_subdomain.cpp
@@ -2568,6 +2568,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) {
 
         }
 
+
         //clear subdomain
         {
             TestDescribeResult(DescribePath(runtime, "/MyRoot"),
@@ -2585,6 +2586,155 @@ Y_UNIT_TEST_SUITE(TSchemeShardSubDomainTest) {
         }
     }
 
+    Y_UNIT_TEST(ColumnSchemeLimitsRejects) {
+        TTestBasicRuntime runtime;
+        TTestEnv env(runtime);
+        ui64 txId = 100;
+
+        TSchemeLimits lowLimits;
+        lowLimits.MaxDepth = 4;
+        lowLimits.MaxPaths = 3;
+        lowLimits.MaxChildrenInDir = 3;
+        lowLimits.MaxAclBytesSize = 25;
+        lowLimits.MaxTableColumns = 3;
+        lowLimits.MaxColumnTableColumns = 3;
+        lowLimits.MaxTableColumnNameLength = 10;
+        lowLimits.MaxTableKeyColumns = 1;
+        lowLimits.MaxShards = 6;
+        lowLimits.MaxShardsInPath = 4;
+        lowLimits.MaxPQPartitions = 20;
+
+
+        //lowLimits.ExtraPathSymbolsAllowed = "!\"#$%&'()*+,-.:;<=>?@[\\]^_`{|}~";
+        SetSchemeshardSchemaLimits(runtime, lowLimits);
+        TestDescribeResult(DescribePath(runtime, "/MyRoot"),
+                           {NLs::PathExist,
+                            NLs::DomainLimitsIs(lowLimits.MaxPaths, lowLimits.MaxShards, lowLimits.MaxPQPartitions)});
+
+        {
+            TestCreateSubDomain(runtime, txId++,  "/MyRoot",
+                                "PlanResolution: 50 "
+                                "Coordinators: 1 "
+                                "Mediators: 1 "
+                                "TimeCastBucketsPerMediator: 2 "
+                                "Name: \"USER_0\""
+                                " DatabaseQuotas {"
+                                "    data_stream_shards_quota: 2"
+                                "    data_stream_reserved_storage_quota: 200000"
+                                "}");
+        }
+
+        //create column tables, column limits
+        {
+            TestMkDir(runtime, txId++, "/MyRoot/USER_0", "C");
+            env.TestWaitNotification(runtime, txId - 1);
+
+            // MaxColumnTableColumns
+            TestCreateColumnTable(runtime, txId++, "/MyRoot/USER_0/C", R"(
+                            Name: "C2"
+                            ColumnShardCount: 1
+                            Schema {
+                                Columns { Name: "RowId" Type: "Uint64", NotNull: true }
+                                Columns { Name: "Value0" Type: "Utf8" }
+                                Columns { Name: "Value1" Type: "Utf8" }
+                                KeyColumnNames: "RowId"
+                                Engine: COLUMN_ENGINE_REPLACING_TIMESERIES
+                            }
+                )", {NKikimrScheme::StatusAccepted});
+            env.TestWaitNotification(runtime, txId - 1);
+
+            TestAlterColumnTable(runtime, txId++, "/MyRoot/USER_0/C", R"(
+                Name: "C2"
+                AlterSchema {
+                    DropColumns {Name: "Value0"}
+                }
+            )", {NKikimrScheme::StatusAccepted});
+            env.TestWaitNotification(runtime, txId - 1);
+
+            TestAlterColumnTable(runtime, txId++, "/MyRoot/USER_0/C", R"(
+                Name: "C2"
+                AlterSchema {
+                    DropColumns {Name: "Value1"}
+                    AddColumns { Name: "Value2" Type: "Utf8" }
+                    AddColumns { Name: "Value3" Type: "Utf8" }
+                    AddColumns { Name: "Value4" Type: "Utf8" }
+                }
+            )", {NKikimrScheme::StatusSchemeError});
+            env.TestWaitNotification(runtime, txId - 1);
+
+            TestCreateColumnTable(runtime, txId++, "/MyRoot/USER_0/C", R"(
+                            Name: "C1"
+                            ColumnShardCount: 1
+                            Schema {
+                                Columns { Name: "RowId" Type: "Uint64", NotNull: true }
+                                Columns { Name: "Value0" Type: "Utf8" }
+                                Columns { Name: "Value1" Type: "Utf8" }
+                                Columns { Name: "Value2" Type: "Utf8" }
+                                KeyColumnNames: "RowId"
+                                Engine: COLUMN_ENGINE_REPLACING_TIMESERIES
+                            }
+                )", {NKikimrScheme::StatusSchemeError});
+
+            TString olapSchema = R"(
+                Name: "OlapStore1"
+                ColumnShardCount: 1
+                SchemaPresets {
+                    Name: "default"
+                    Schema {
+                        Columns { Name: "timestamp" Type: "Timestamp" NotNull: true }
+                        Columns { Name: "data" Type: "Utf8" }
+                        KeyColumnNames: "timestamp"
+                        Engine: COLUMN_ENGINE_REPLACING_TIMESERIES
+                    }
+                }
+            )";
+
+            TestCreateOlapStore(runtime, txId++, "/MyRoot", olapSchema, {NKikimrScheme::StatusAccepted});
+            env.TestWaitNotification(runtime, txId - 1);
+
+            TString olapSchemaBig = R"(
+                Name: "OlapStoreBig"
+                ColumnShardCount: 1
+                SchemaPresets {
+                    Name: "default"
+                    Schema {
+                        Columns { Name: "timestamp" Type: "Timestamp" NotNull: true }
+                        Columns { Name: "data" Type: "Utf8" }
+                        Columns { Name: "data2" Type: "Utf8" }
+                        Columns { Name: "data3" Type: "Utf8" }
+                        KeyColumnNames: "timestamp"
+                        Engine: COLUMN_ENGINE_REPLACING_TIMESERIES
+                    }
+                }
+            )";
+
+            TestCreateOlapStore(runtime, txId++, "/MyRoot", olapSchemaBig, {NKikimrScheme::StatusSchemeError});
+            env.TestWaitNotification(runtime, txId - 1);
+
+            TestAlterOlapStore(runtime, txId++, "/MyRoot", R"(
+                Name: "OlapStore1"
+                AlterSchemaPresets {
+                    Name: "default"
+                    AlterSchema {
+                        AddColumns { Name: "comment" Type: "Utf8" }
+                    }
+                }
+            )", {NKikimrScheme::StatusAccepted});
+            env.TestWaitNotification(runtime, txId - 1);
+
+            TestAlterOlapStore(runtime, txId++, "/MyRoot", R"(
+                Name: "OlapStore1"
+                AlterSchemaPresets {
+                    Name: "default"
+                    AlterSchema {
+                        AddColumns { Name: "comment2" Type: "Utf8" }
+                    }
+                }
+            )", {NKikimrScheme::StatusSchemeError});
+            env.TestWaitNotification(runtime, txId - 1);
+        }
+    }
+
     Y_UNIT_TEST(SchemeLimitsRejectsWithIndexedTables) {
         TTestBasicRuntime runtime;
         TTestEnv env(runtime);
diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema
@@ -1454,6 +1454,11 @@
                 "ColumnId": 31,
                 "ColumnName": "ServerlessComputeResourcesMode",
                 "ColumnType": "Uint32"
+            },
+            {
+                "ColumnId": 32,
+                "ColumnName": "ColumnTableColumnsLimit",
+                "ColumnType": "Uint64"
             }
         ],
         "ColumnsDropped": [],
@@ -1490,7 +1495,8 @@
                     28,
                     29,
                     30,
-                    31
+                    31,
+                    32
                 ],
                 "RoomID": 0,
                 "Codec": 0,

Original file line number	Diff line number	Diff line change
`@@ -190,6 +190,8 @@ message TSchemeLimits {`
`190`	`190`
`191`	`191`	`optional uint64 MaxExports = 16;`
`192`	`192`	`optional uint64 MaxImports = 17;`
	`193`	`+`
	`194`	`+ optional uint64 MaxColumnTableColumns = 18;`
`193`	`195`	`}`
`194`	`196`
`195`	`197`	`message TEvInitTenantSchemeShard {`