not null constraint fix for column addition

ydb/core/kqp/provider/yql_kikimr_exec.cpp

@@ -1086,6 +1086,9 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformer<TKi
                         auto dataType = actualType->Cast<TDataExprType>();
                         SetColumnType(*add_column->mutable_type(), TString(dataType->GetName()), notNull);
 
+                        ::NKikimrIndexBuilder::TColumnBuildSetting* columnBuild = nullptr;
+                        bool hasDefaultValue = false;
+                        bool hasNotNull = false;
                         if (columnTuple.Size() > 2) {
                             auto columnConstraints = columnTuple.Item(2).Cast<TCoNameValueTuple>();
                             for(const auto& constraint: columnConstraints.Value().Cast<TCoNameValueTupleList>()) {
@@ -1094,13 +1097,32 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformer<TKi
                                         "Column addition with serial data type is unsupported"));
                                     return SyncError();
                                 } else if (constraint.Name().Value() == "default") {
-                                    auto columnBuild = indexBuildSettings.mutable_column_build_operation()->add_column();
+                                    if (columnBuild == nullptr) {
+                                        columnBuild = indexBuildSettings.mutable_column_build_operation()->add_column();
+                                    }
+
                                     columnBuild->SetColumnName(TString(columnName));
                                     FillLiteralProto(constraint.Value().Cast<TCoDataCtor>(), *columnBuild->mutable_default_from_literal());
+                                    hasDefaultValue = true;
+                                } else if (constraint.Name().Value() == "not_null") {
+                                    if (columnBuild == nullptr) {
+                                        columnBuild = indexBuildSettings.mutable_column_build_operation()->add_column();
+                                    }
+
+                                    columnBuild->SetNotNull(true);
+                                    hasNotNull = true;
                                 }
                             }
                         }
 
+                        if (hasNotNull && !hasDefaultValue) {
+                            ctx.AddError(
+                                YqlIssue(ctx.GetPosition(columnTuple.Pos()),
+                                    TIssuesIds::KIKIMR_BAD_REQUEST,
+                                    "Cannot add not null column without default value"));
+                            return SyncError();
+                        }
+
                         if (columnTuple.Size() > 3) {
                             auto families = columnTuple.Item(3).Cast<TCoAtomList>();
                             if (families.Size() > 1) {
@@ -1112,6 +1134,12 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformer<TKi
                             for (auto family : families) {
                                 add_column->set_family(TString(family.Value()));
                             }
+
+                            if (columnBuild) {
+                                for (auto family : families) {
+                                    columnBuild->SetFamily(TString(family.Value()));
+                                }
+                            }
                         }
                     }
                 } else if (name == "dropColumns") {

ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp

@@ -291,6 +291,7 @@ Y_UNIT_TEST_SUITE(KqpConstraints) {
     Y_UNIT_TEST(AlterTableAddColumnWithDefaultValue) {
         NKikimrConfig::TAppConfig appConfig;
         appConfig.MutableTableServiceConfig()->SetEnableSequences(false);
+        appConfig.MutableFeatureFlags()->SetEnableAddColumsWithDefaults(true);
         auto serverSettings = TKikimrSettings().SetAppConfig(appConfig);
         TKikimrRunner kikimr(serverSettings);
         auto db = kikimr.GetTableClient();
@@ -588,6 +589,8 @@ Y_UNIT_TEST_SUITE(KqpConstraints) {
         appConfig.MutableTableServiceConfig()->SetEnableSequences(false);
         appConfig.MutableTableServiceConfig()->SetEnableColumnsWithDefault(true);
 
+        appConfig.MutableFeatureFlags()->SetEnableAddColumsWithDefaults(true);
+
         TKikimrRunner kikimr(TKikimrSettings().SetPQConfig(DefaultPQConfig()).SetAppConfig(appConfig));
         auto db = kikimr.GetTableClient();
         auto session = db.CreateSession().GetValueSync().GetSession();
@@ -661,7 +664,7 @@ Y_UNIT_TEST_SUITE(KqpConstraints) {
 
         fCompareTable(R"(
             [
-                [[1u];["Old"];[1]];[[2u];["New"];[1]]
+                [[1u];["Old"];1];[[2u];["New"];1]
             ]
         )");
 
@@ -672,7 +675,7 @@ Y_UNIT_TEST_SUITE(KqpConstraints) {
 
         fCompareTable(R"(
             [
-                [[1u];["Old"];[1]];[[2u];["New"];[2]]
+                [[1u];["Old"];1];[[2u];["New"];2]
             ]
         )");
 
@@ -686,7 +689,7 @@ Y_UNIT_TEST_SUITE(KqpConstraints) {
 
         fCompareTable(R"(
             [
-                [[1u];["Old"];[1]];[[2u];["OldNew"];[2]];[[3u];["BrandNew"];[1]]
+                [[1u];["Old"];1];[[2u];["OldNew"];2];[[3u];["BrandNew"];1]
             ]
         )");
 

ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp

@@ -5553,7 +5553,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) {
         {
             auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "`ADD COLUMN new_column Uint64 NOT NULL;";
             auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync();
-            UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SCHEME_ERROR, alterResult.GetIssues().ToString());
+            UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::BAD_REQUEST, alterResult.GetIssues().ToString());
         }
     }
 

ydb/core/protos/feature_flags.proto

@@ -127,4 +127,5 @@ message TFeatureFlags {
     optional bool EnableViews = 112 [default = false];
     optional bool EnableServerlessExclusiveDynamicNodes = 113 [default = false];
     optional bool EnableAccessServiceBulkAuthorization = 114 [default = false];
+    optional bool EnableAddColumsWithDefaults = 115 [ default = false];
 }

ydb/core/protos/index_builder.proto

@@ -12,6 +12,8 @@ option java_package = "ru.yandex.kikimr.proto";
 message TColumnBuildSetting {
     optional string ColumnName = 1;
     optional Ydb.TypedValue default_from_literal = 2;
+    optional bool NotNull = 3;
+    optional string Family = 4;
 }
 
 message TColumnBuildSettings {

ydb/core/testlib/basics/feature_flags.h

@@ -56,6 +56,7 @@ class TTestFeatureFlagsHolder {
     FEATURE_FLAG_SETTER(EnableTablePgTypes)
     FEATURE_FLAG_SETTER(EnableServerlessExclusiveDynamicNodes)
     FEATURE_FLAG_SETTER(EnableAccessServiceBulkAuthorization)
+    FEATURE_FLAG_SETTER(EnableAddColumsWithDefaults)
 
     #undef FEATURE_FLAG_SETTER
 };

ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp

@@ -87,11 +87,19 @@ TTableInfo::TAlterDataPtr ParseParams(const TPath& path, TTableInfo::TPtr table,
 
     // Ignore column ids if they were passed by user!
     for (auto& col : *copyAlter.MutableColumns()) {
-        if (col.GetNotNull()) {
-            errStr = Sprintf("Not null columns is not supported for alter command");
+        bool hasDefault = col.HasDefaultFromLiteral();
+        if (hasDefault && !context.SS->EnableAddColumsWithDefaults) {
+            errStr = Sprintf("Column addition with default value is not supported now.");
             status = NKikimrScheme::StatusInvalidParameter;
             return nullptr;
         }
+
+        if (col.GetNotNull() && !hasDefault) {
+            errStr = Sprintf("Not null columns without defaults are not supported.");
+            status = NKikimrScheme::StatusInvalidParameter;
+            return nullptr;
+        }
+
         col.ClearId();
     }
 

ydb/core/tx/schemeshard/schemeshard_build_index.cpp

@@ -68,7 +68,9 @@ void TSchemeShard::PersistCreateBuildIndex(NIceDb::TNiceDb& db, const TIndexBuil
         db.Table<Schema::BuildColumnOperationSettings>().Key(info->Id, i).Update(
             NIceDb::TUpdate<Schema::BuildColumnOperationSettings::ColumnName>(info->BuildColumns[i].ColumnName),
             NIceDb::TUpdate<Schema::BuildColumnOperationSettings::DefaultFromLiteral>(
-                TString(info->BuildColumns[i].DefaultFromLiteral.SerializeAsString()))
+                TString(info->BuildColumns[i].DefaultFromLiteral.SerializeAsString())),
+            NIceDb::TUpdate<Schema::BuildColumnOperationSettings::NotNull>(info->BuildColumns[i].NotNull),
+            NIceDb::TUpdate<Schema::BuildColumnOperationSettings::FamilyName>(info->BuildColumns[i].FamilyName)
         );
     }
 }

ydb/core/tx/schemeshard/schemeshard_build_index__create.cpp

@@ -161,8 +161,11 @@ class TSchemeShard::TIndexBuilder::TTxCreate: public TSchemeShard::TIndexBuilder
             buildInfo->BuildColumns.reserve(settings.column_build_operation().column_size());
             for(int i = 0; i < settings.column_build_operation().column_size(); i++) {
                 const auto& colInfo = settings.column_build_operation().column(i);
+                bool notNull = colInfo.HasNotNull() && colInfo.GetNotNull();
+                TString familyName = colInfo.HasFamily() ? colInfo.GetFamily() : "";
                 buildInfo->BuildColumns.push_back(
-                    TIndexBuildInfo::TColumnBuildInfo(colInfo.GetColumnName(), colInfo.default_from_literal()));
+                    TIndexBuildInfo::TColumnBuildInfo(
+                        colInfo.GetColumnName(), colInfo.default_from_literal(), notNull, familyName));
             }
         }
 

ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp

@@ -98,6 +98,15 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> AlterMainTablePropose(
             col->SetType(NScheme::TypeName(typeInfo, typeMod));
             col->SetName(colInfo.ColumnName);
             col->MutableDefaultFromLiteral()->CopyFrom(colInfo.DefaultFromLiteral);
+
+            if (!colInfo.FamilyName.empty()) {
+                col->SetFamilyName(colInfo.FamilyName);
+            }
+
+            if (colInfo.NotNull) {
+                col->SetNotNull(colInfo.NotNull);
+            }
+
         }
 
     } else {

ydb/core/tx/schemeshard/schemeshard_impl.cpp

@@ -4271,6 +4271,7 @@ void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) {
     EnableStatistics = appData->FeatureFlags.GetEnableStatistics();
     EnableTablePgTypes = appData->FeatureFlags.GetEnableTablePgTypes();
     EnableServerlessExclusiveDynamicNodes = appData->FeatureFlags.GetEnableServerlessExclusiveDynamicNodes();
+    EnableAddColumsWithDefaults = appData->FeatureFlags.GetEnableAddColumsWithDefaults();
 
     ConfigureCompactionQueues(appData->CompactionConfig, ctx);
     ConfigureStatsBatching(appData->SchemeShardConfig, ctx);
@@ -6748,6 +6749,7 @@ void TSchemeShard::ApplyConsoleConfigs(const NKikimrConfig::TFeatureFlags& featu
     EnableStatistics = featureFlags.GetEnableStatistics();
     EnableTablePgTypes = featureFlags.GetEnableTablePgTypes();
     EnableServerlessExclusiveDynamicNodes = featureFlags.GetEnableServerlessExclusiveDynamicNodes();
+    EnableAddColumsWithDefaults = featureFlags.GetEnableAddColumsWithDefaults();
 }
 
 void TSchemeShard::ConfigureStatsBatching(const NKikimrConfig::TSchemeShardConfig& config, const TActorContext& ctx) {

ydb/core/tx/schemeshard/schemeshard_impl.h

@@ -271,6 +271,7 @@ class TSchemeShard
     bool EnableStatistics = false;
     bool EnableTablePgTypes = false;
     bool EnableServerlessExclusiveDynamicNodes = false;
+    bool EnableAddColumsWithDefaults = false;
 
     TShardDeleter ShardDeleter;
 

ydb/core/tx/schemeshard/schemeshard_info_types.h

@@ -3007,22 +3007,30 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
     struct TColumnBuildInfo {
         TString ColumnName;
         Ydb::TypedValue DefaultFromLiteral;
+        bool NotNull = false;
+        TString FamilyName;
 
-        TColumnBuildInfo(const TString& name, const TString& serializedLiteral)
+        TColumnBuildInfo(const TString& name, const TString& serializedLiteral, bool notNull, const TString& familyName)
             : ColumnName(name)
+            , NotNull(notNull)
+            , FamilyName(familyName)
         {
             Y_ABORT_UNLESS(DefaultFromLiteral.ParseFromString(serializedLiteral));
         }
 
-        TColumnBuildInfo(const TString& name, const Ydb::TypedValue& defaultFromLiteral)
+        TColumnBuildInfo(const TString& name, const Ydb::TypedValue& defaultFromLiteral, bool notNull, const TString& familyName)
             : ColumnName(name)
             , DefaultFromLiteral(defaultFromLiteral)
+            , NotNull(notNull)
+            , FamilyName(familyName)
         {
         }
 
         void SerializeToProto(NKikimrIndexBuilder::TColumnBuildSetting* setting) const {
             setting->SetColumnName(ColumnName);
             setting->mutable_default_from_literal()->CopyFrom(DefaultFromLiteral);
+            setting->SetNotNull(NotNull);
+            setting->SetFamily(FamilyName);
         }
     };
 
@@ -3140,7 +3148,9 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
     void AddBuildColumnInfo(const TRow& row){
         TString columnName = row.template GetValue<Schema::BuildColumnOperationSettings::ColumnName>();
         TString defaultFromLiteral = row.template GetValue<Schema::BuildColumnOperationSettings::DefaultFromLiteral>();
-        BuildColumns.push_back(TColumnBuildInfo(columnName, defaultFromLiteral));
+        bool notNull = row.template GetValue<Schema::BuildColumnOperationSettings::NotNull>();
+        TString familyName = row.template GetValue<Schema::BuildColumnOperationSettings::FamilyName>();
+        BuildColumns.push_back(TColumnBuildInfo(columnName, defaultFromLiteral, notNull, familyName));
     }
 
     template<class TRowSetType>

ydb/core/tx/schemeshard/schemeshard_schema.h

@@ -1726,13 +1726,17 @@ struct Schema : NIceDb::Schema {
         struct ColumnNo : Column<2, NScheme::NTypeIds::Uint64> {};
         struct ColumnName : Column<3, NScheme::NTypeIds::Utf8> {};
         struct DefaultFromLiteral : Column<4, NScheme::NTypeIds::String> {};
+        struct NotNull : Column<5, NScheme::NTypeIds::Bool> {};
+        struct FamilyName : Column<6, NScheme::NTypeIds::String> {};
 
         using TKey = TableKey<Id, ColumnNo>;
         using TColumns = TableColumns<
             Id,
             ColumnNo,
             ColumnName,
-            DefaultFromLiteral
+            DefaultFromLiteral,
+            NotNull,
+            FamilyName
         >;
     };
 

ydb/core/tx/schemeshard/ut_column_build/ut_column_build.cpp

@@ -12,7 +12,7 @@ using namespace NSchemeShardUT_Private;
 Y_UNIT_TEST_SUITE(ColumnBuildTest) {
     Y_UNIT_TEST(AlreadyExists) {
         TTestBasicRuntime runtime;
-        TTestEnv env(runtime);
+        TTestEnv env(runtime, TTestEnvOptions().EnableAddColumsWithDefaults(true));
         ui64 txId = 100;
 
         TestCreateExtSubDomain(runtime, ++txId,  "/MyRoot",
@@ -117,7 +117,7 @@ Y_UNIT_TEST_SUITE(ColumnBuildTest) {
 
     Y_UNIT_TEST(InvalidValue) {
         TTestBasicRuntime runtime;
-        TTestEnv env(runtime);
+        TTestEnv env(runtime, TTestEnvOptions().EnableAddColumsWithDefaults(true));
         ui64 txId = 100;
 
         TestCreateExtSubDomain(runtime, ++txId,  "/MyRoot",
@@ -230,7 +230,7 @@ Y_UNIT_TEST_SUITE(ColumnBuildTest) {
 
     Y_UNIT_TEST(BuildColumnDoesnotRestoreDeletedRows) {
         TTestBasicRuntime runtime(1, false);
-        TTestEnv env(runtime);
+        TTestEnv env(runtime, TTestEnvOptions().EnableAddColumsWithDefaults(true));
         ui64 txId = 100;
 
         TestCreateExtSubDomain(runtime, ++txId,  "/MyRoot",
@@ -400,7 +400,7 @@ Y_UNIT_TEST_SUITE(ColumnBuildTest) {
 
     Y_UNIT_TEST(BaseCase) {
         TTestBasicRuntime runtime;
-        TTestEnv env(runtime);
+        TTestEnv env(runtime, TTestEnvOptions().EnableAddColumsWithDefaults(true));
         ui64 txId = 100;
 
         TestCreateExtSubDomain(runtime, ++txId,  "/MyRoot",
@@ -545,7 +545,7 @@ Y_UNIT_TEST_SUITE(ColumnBuildTest) {
 
     Y_UNIT_TEST(CancelBuild) {
         TTestBasicRuntime runtime;
-        TTestEnv env(runtime);
+        TTestEnv env(runtime, TTestEnvOptions().EnableAddColumsWithDefaults(true));
         ui64 txId = 100;
 
         // Just create main table

ydb/core/tx/schemeshard/ut_helpers/test_env.cpp

@@ -536,6 +536,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe
     app.SetEnableChangefeedDebeziumJsonFormat(opts.EnableChangefeedDebeziumJsonFormat_);
     app.SetEnableTablePgTypes(opts.EnableTablePgTypes_);
     app.SetEnableServerlessExclusiveDynamicNodes(opts.EnableServerlessExclusiveDynamicNodes_);
+    app.SetEnableAddColumsWithDefaults(opts.EnableAddColumsWithDefaults_);
 
     app.ColumnShardConfig.SetDisabledOnSchemeShard(false);
 

ydb/core/tx/schemeshard/ut_helpers/test_env.h

@@ -58,6 +58,7 @@ namespace NSchemeShardUT_Private {
         OPTION(std::optional<bool>, EnableChangefeedDebeziumJsonFormat, std::nullopt);
         OPTION(std::optional<bool>, EnableTablePgTypes, std::nullopt);
         OPTION(std::optional<bool>, EnableServerlessExclusiveDynamicNodes, std::nullopt);
+        OPTION(std::optional<bool>, EnableAddColumsWithDefaults, std::nullopt);
 
         #undef OPTION
     };

ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema

@@ -7145,6 +7145,16 @@
                 "ColumnId": 4,
                 "ColumnName": "DefaultFromLiteral",
                 "ColumnType": "String"
+            },
+            {
+                "ColumnId": 5,
+                "ColumnName": "NotNull",
+                "ColumnType": "Bool"
+            },
+            {
+                "ColumnId": 6,
+                "ColumnName": "FamilyName",
+                "ColumnType": "String"
             }
         ],
         "ColumnsDropped": [],
@@ -7154,7 +7164,9 @@
                     1,
                     2,
                     3,
-                    4
+                    4,
+                    5,
+                    6
                 ],
                 "RoomID": 0,
                 "Codec": 0,