Merge branch 'stable-24-4' into docs-backport-18129-to-stable-24-4

Author: Anton Bobkov

ydb/core/kafka_proxy/kafka_connection.cpp

@@ -101,7 +101,7 @@ class TKafkaConnection: public TActorBootstrapped<TKafkaConnection>, public TNet
 
     void Bootstrap() {
         Context->ConnectionId = SelfId();
-        Context->RequireAuthentication = NKikimr::AppData()->EnforceUserTokenRequirement;
+        Context->RequireAuthentication = NKikimr::AppData()->EnforceUserTokenRequirement || NKikimr::AppData()->PQConfig.GetRequireCredentialsInNewProtocol();
         // if no authentication required, then we can use local database as our target
         if (!Context->RequireAuthentication) {
             Context->DatabasePath = NKikimr::AppData()->TenantName;
@@ -452,7 +452,7 @@ class TKafkaConnection: public TActorBootstrapped<TKafkaConnection>, public TNet
             return;
         }
 
-        Context->RequireAuthentication = NKikimr::AppData()->EnforceUserTokenRequirement;
+        Context->RequireAuthentication = NKikimr::AppData()->EnforceUserTokenRequirement || NKikimr::AppData()->PQConfig.GetRequireCredentialsInNewProtocol();
         Context->UserToken = event->UserToken;
         Context->DatabasePath = event->DatabasePath;
         Context->AuthenticationStep = authStep;

ydb/core/mind/bscontroller/impl.h

@@ -280,6 +280,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
             const ui64 allocatedSizeBefore = Metrics.GetAllocatedSize();
             const ui32 prevStatusFlags = Metrics.GetStatusFlags();
             Metrics.MergeFrom(vDiskMetrics);
+            Metrics.DiscardUnknownFields();
             MetricsDirty = true;
             UpdateVDiskMetrics();
             *allocatedSizeIncrementPtr = Metrics.GetAllocatedSize() - allocatedSizeBefore;

ydb/core/tablet/resource_broker.cpp

@@ -1275,7 +1275,7 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig()
     const ui64 CSGeneralCompactionMemoryLimit = NOlap::TGlobalLimits::GeneralCompactionMemoryLimit;
     const ui64 CSScanMemoryLimit = NOlap::TGlobalLimits::ScanMemoryLimit;
 
-    const ui64 TotalCPU = 20;
+    const ui64 TotalCPU = 256; // means unlimited
     const ui64 TotalMemory = 16ULL << 30;
 
     static_assert(KqpRmQueueMemory < TotalMemory);

ydb/core/tablet/resource_broker_ut.cpp

@@ -15,13 +15,13 @@ namespace NKikimr {
 using namespace NKikimrResourceBroker;
 using namespace NResourceBroker;
 
-static void SetupLogging(TTestActorRuntime& runtime) {
+static void SetupLogging(TTestActorRuntime& runtime)
+{
     NActors::NLog::EPriority priority = ENABLE_DETAILED_RESOURCE_BROKER_LOG ? NLog::PRI_DEBUG : NLog::PRI_ERROR;
     runtime.SetLogPriority(NKikimrServices::RESOURCE_BROKER, priority);
 }
 
-static NKikimrResourceBroker::TResourceBrokerConfig
-MakeTestConfig()
+static NKikimrResourceBroker::TResourceBrokerConfig MakeTestConfig()
 {
     NKikimrResourceBroker::TResourceBrokerConfig config;
 
@@ -71,16 +71,14 @@ MakeTestConfig()
     return config;
 }
 
-static void
-WaitForBootstrap(TTestActorRuntime &runtime)
+static void WaitForBootstrap(TTestActorRuntime &runtime)
 {
     TDispatchOptions options;
     options.FinalEvents.emplace_back(TEvents::TSystem::Bootstrap, 1);
     UNIT_ASSERT(runtime.DispatchEvents(options));
 }
 
-static void
-SubmitTask(TTestActorRuntime &runtime, TActorId broker, TActorId sender,
+static void SubmitTask(TTestActorRuntime &runtime, TActorId broker, TActorId sender,
            ui64 id, ui64 cpu, ui64 memory, const TString &type,
            ui32 priority, TIntrusivePtr<TThrRefBase> cookie = nullptr)
 {
@@ -94,8 +92,7 @@ SubmitTask(TTestActorRuntime &runtime, TActorId broker, TActorId sender,
     runtime.Send(new IEventHandle(broker, sender, event.Release()));
 }
 
-static void
-UpdateTask(TTestActorRuntime &runtime, TActorId broker, TActorId sender,
+static void UpdateTask(TTestActorRuntime &runtime, TActorId broker, TActorId sender,
            ui64 id, ui64 cpu, ui64 memory, ui32 priority, const TString &type,
            bool resubmit = false)
 {
@@ -107,8 +104,7 @@ UpdateTask(TTestActorRuntime &runtime, TActorId broker, TActorId sender,
     runtime.Send(new IEventHandle(broker, sender, event.Release()));
 }
 
-static void
-UpdateTaskCookie(TTestActorRuntime &runtime, TActorId broker, TActorId sender,
+static void UpdateTaskCookie(TTestActorRuntime &runtime, TActorId broker, TActorId sender,
                  ui64 id, TIntrusivePtr<TThrRefBase> cookie)
 {
     TAutoPtr<TEvResourceBroker::TEvUpdateTaskCookie> event
@@ -117,17 +113,15 @@ UpdateTaskCookie(TTestActorRuntime &runtime, TActorId broker, TActorId sender,
     runtime.Send(new IEventHandle(broker, sender, event.Release()));
 }
 
-static void
-RemoveTask(TTestActorRuntime &runtime, TActorId broker, TActorId sender, ui64 id)
+static void RemoveTask(TTestActorRuntime &runtime, TActorId broker, TActorId sender, ui64 id)
 {
     TAutoPtr<TEvResourceBroker::TEvRemoveTask> event
         = new TEvResourceBroker::TEvRemoveTask(id);
 
     runtime.Send(new IEventHandle(broker, sender, event.Release()));
 }
 
-static void
-FinishTask(TTestActorRuntime &runtime, TActorId broker, TActorId sender, ui64 id)
+static void FinishTask(TTestActorRuntime &runtime, TActorId broker, TActorId sender, ui64 id)
 {
     TAutoPtr<TEvResourceBroker::TEvFinishTask> event
         = new TEvResourceBroker::TEvFinishTask(id);
@@ -147,8 +141,7 @@ WaitForResourceAllocation(TTestActorRuntime &runtime, ui64 id,
     return reply->TaskId;
 }
 
-static void
-WaitForError(TTestActorRuntime &runtime, ui64 id, TEvResourceBroker::TStatus::ECode code,
+static void WaitForError(TTestActorRuntime &runtime, ui64 id, TEvResourceBroker::TStatus::ECode code,
              TIntrusivePtr<TThrRefBase> cookie = nullptr)
 {
     TAutoPtr<IEventHandle> handle;
@@ -158,8 +151,7 @@ WaitForError(TTestActorRuntime &runtime, ui64 id, TEvResourceBroker::TStatus::EC
     UNIT_ASSERT_VALUES_EQUAL(reply->Cookie, cookie);
 }
 
-static void
-CheckCounters(::NMonitoring::TDynamicCounterPtr counters, const TString &group, const TString &name,
+static void CheckCounters(::NMonitoring::TDynamicCounterPtr counters, const TString &group, const TString &name,
               ui64 cpu, ui64 memory, ui64 finished, ui64 enqueued, ui64 infly)
 {
     auto g = counters->GetSubgroup(group, name);
@@ -185,14 +177,14 @@ static void CheckConfigure(TTestActorRuntime &runtime, TActorId broker, TActorId
     UNIT_ASSERT_VALUES_EQUAL((int)rec.GetSuccess(), (int)success);
 }
 
-static
-TIntrusivePtr<IResourceBroker> GetInstantResourceBroker(TTestActorRuntime &runtime, TActorId broker, TActorId sender) {
+static TIntrusivePtr<IResourceBroker> GetInstantResourceBroker(TTestActorRuntime &runtime, TActorId broker, TActorId sender) {
     runtime.Send(new IEventHandle(broker, sender, new TEvResourceBroker::TEvResourceBrokerRequest));
     auto answer = runtime.GrabEdgeEvent<TEvResourceBroker::TEvResourceBrokerResponse>(sender);
     return answer->Get()->ResourceBroker;
 }
 
 Y_UNIT_TEST_SUITE(TResourceBroker) {
+
     Y_UNIT_TEST(TestErrors) {
         TTestBasicRuntime runtime;
         SetupTabletServices(runtime);
@@ -894,6 +886,7 @@ Y_UNIT_TEST_SUITE(TResourceBroker) {
 };
 
 Y_UNIT_TEST_SUITE(TResourceBrokerInstant) {
+
     Y_UNIT_TEST(Test) {
         TTestBasicRuntime runtime;
         SetupTabletServices(runtime);
@@ -1007,6 +1000,7 @@ Y_UNIT_TEST_SUITE(TResourceBrokerInstant) {
             CheckCounters(counters, "queue", "total", 210, 320, 1, 0, 2);
         }
     }
+
 };
 
 Y_UNIT_TEST_SUITE(TResourceBrokerConfig) {
@@ -1130,6 +1124,21 @@ Y_UNIT_TEST_SUITE(TResourceBrokerConfig) {
         UNIT_ASSERT_VALUES_EQUAL(config.ShortDebugString(), "ResourceLimit { Cpu: 20 Memory: 2048 }");
     }
 
+    Y_UNIT_TEST(DefaultConfig) {
+        auto config = MakeDefaultConfig();
+
+        Cerr << config.DebugString() << Endl;
+
+        ui64 queuesCpu = 0;
+        for (const auto& q : config.GetQueues()) {
+            queuesCpu += q.GetLimit().GetCpu();
+        }
+        Cerr << "Total queues cpu: " << queuesCpu;
+
+        // see https://github.com/ydb-platform/ydb/issues/18513
+        UNIT_ASSERT_LE(queuesCpu, config.GetResourceLimit().GetCpu());
+    }
+
 } // TResourceBrokerConfig
 
 } // NKikimr

ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.h

@@ -50,7 +50,10 @@ class TStatsIteratorBase: public TScanIteratorBase {
 
             // Leave only requested columns
             auto resultBatch = NArrow::TColumnOperator().Adapt(originalBatch, ResultSchema).DetachResult();
-            NArrow::TStatusValidator::Validate(ReadMetadata->GetProgram().ApplyProgram(resultBatch));
+            auto applyConclusion = ReadMetadata->GetProgram().ApplyProgram(resultBatch);
+            if (!applyConclusion.ok()) {
+                return TConclusionStatus::Fail(applyConclusion.ToString());
+            }
             if (resultBatch->num_rows() == 0) {
                 continue;
             }

ydb/core/tx/schemeshard/schemeshard__table_stats.cpp

@@ -407,9 +407,11 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId,
     const TTableIndexInfo* index = Self->Indexes.Value(pathElement->ParentPathId, nullptr).Get();
     const TTableInfo* mainTableForIndex = Self->GetMainTableForIndex(pathId);
 
+    TString errStr;
     const auto forceShardSplitSettings = Self->SplitSettings.GetForceShardSplitSettings();
     TVector<TShardIdx> shardsToMerge;
     if ((!index || index->State == NKikimrSchemeOp::EIndexStateReady)
+        && Self->CheckInFlightLimit(NKikimrSchemeOp::ESchemeOpSplitMergeTablePartitions, errStr)
         && table->CheckCanMergePartitions(Self->SplitSettings, forceShardSplitSettings, shardIdx, shardsToMerge, mainTableForIndex)
     ) {
         TTxId txId = Self->GetCachedTxId(ctx);

ydb/docs/en/core/concepts/topic.md

@@ -129,7 +129,7 @@ A source ID is an arbitrary string up to 2048 characters long. This is usually t
 #### Sample source IDs {#source-id-examples}
 
 | Type | ID | Description |
---- | --- | ---
+| --- | --- | --- |
 | File | Server ID | Files are used to store application logs. In this case, it's convenient to use the server ID as a source ID. |
 | User actions | ID of the class of user actions, such as "viewing a page", "making a purchase", and so on. | It's important to handle user actions in the order they were performed by the user. At the same time, there is no need to handle every single user action in one application. In this case, it's convenient to group user actions by class. |
 
@@ -140,7 +140,7 @@ A message group ID is an arbitrary string up to 2048 characters long. This is us
 #### Sample message group IDs {#group-id-examples}
 
 | Type | ID | Description |
---- | --- | ---
+| --- | --- | --- |
 | File | Full file path | All data from the server and the file it hosts will be sent to the same partition. |
 | User actions | User ID | It's important to handle user actions in the order they were performed. In this case, it's convenient to use the user ID as a source ID. |
 
@@ -153,9 +153,9 @@ Sequence numbers are not used if [no-deduplication mode](#no-dedup) is enabled.
 ### Sample message sequence numbers {#seqno-examples}
 
 | Type | Example | Description |
---- | --- | ---
+| --- | --- | --- |
 | File | Offset of transferred data from the beginning of a file | You can't delete lines from the beginning of a file, since this will lead to skipping some data as duplicates or losing some data. |
-| DB table | Auto-increment record ID |
+| DB table | Auto-increment record ID | |
 
 ## Message retention period {#retention-time}
 
@@ -168,13 +168,13 @@ When transferring data, the producer app indicates that a message can be compres
 Supported codecs are explicitly listed in each topic. When making an attempt to write data to a topic with a codec that is not supported, a write error occurs.
 
 | Codec | Description |
---- | ---
+| --- | --- |
 | `raw` | No compression. |
 | `gzip` | [Gzip](https://en.wikipedia.org/wiki/Gzip) compression. |
 {% if audience != "external" %}
-`lzop` | [lzop](https://en.wikipedia.org/wiki/Lzop) compression.
+| `lzop` | [lzop](https://en.wikipedia.org/wiki/Lzop) compression. |
 {% endif %}
-`zstd` | [zstd](https://en.wikipedia.org/wiki/Zstd) compression.
+| `zstd` | [zstd](https://en.wikipedia.org/wiki/Zstd) compression. |
 
 ## Consumer {#consumer}
 

ydb/docs/en/core/contributor/documentation/genres.md

@@ -1,8 +1,8 @@
-## {{ ydb-short-name }} documentation genres
+# {{ ydb-short-name }} documentation genres
 
 This article complements [{#T}](style-guide.md) by describing the main genres used in {{ ydb-short-name }} documentation. Understanding these genres helps contributors place new content in the appropriate section and maintain a consistent structure.
 
-### Theory {#theory}
+## Theory {#theory}
 
 **Primary goal for the reader:** build a solid knowledge foundation by understanding the fundamental concepts, architecture, and principles behind {{ ydb-short-name }}.
 
@@ -15,7 +15,7 @@ This article complements [{#T}](style-guide.md) by describing the main genres us
 
 Theory documentation is primarily found in the ["Concepts"](../../concepts/index.md) section but also appears in role-specific folders when the theoretical information is relevant only to a particular audience.
 
-### Guide {#guide}
+## Guide {#guide}
 
 **Primary goal for the reader:** accomplish a specific practical task or implement a particular solution with {{ ydb-short-name }} by following instructions.
 
@@ -29,7 +29,7 @@ Guides are practical, step-by-step instructions that help users accomplish a spe
 
 Guides are primarily found in role-specific folders like ["For DevOps"](../../devops/index.md), ["For Developers"](../../dev/index.md), and ["For Security Engineers"](../../security/index.md), as well as in the ["Troubleshooting"](../../troubleshooting/index.md) section.
 
-### Reference {#reference}
+## Reference {#reference}
 
 **Primary goal for the reader:** find additional information about a specific niche topic related to {{ ydb-short-name }}.
 
@@ -43,7 +43,7 @@ Reference documentation provides comprehensive, detailed information about {{ yd
 
 Reference documentation is designed to be found as needed and is the most detailed level of documentation. It's particularly useful when users need specific information about functions, settings, or keywords. This content is primarily found in the ["Reference"](../../reference/index.md) section.
 
-### FAQ {#faq}
+## FAQ {#faq}
 
 **Primary goal for the reader:** quickly find answers to common questions encountered when working with {{ ydb-short-name }}.
 
@@ -57,7 +57,7 @@ Frequently Asked Questions (FAQ) documentation answers common questions about {{
 
 FAQ content is primarily found in the ["Questions and answers"](../../faq/index.md) section and is designed to help users who are searching for specific solutions to common situations.
 
-### Recipe {#recipe}
+## Recipe {#recipe}
 
 **Primary goal for the reader:** implement a specific, focused solution to a common issue or use case with {{ ydb-short-name }}.
 
@@ -71,7 +71,7 @@ Recipes are short, focused mini-guides that demonstrate how to accomplish specif
 
 Recipes are primarily found in the ["Recipes"](../../recipes/index.md) section, though similar content may also appear in role-specific folders.
 
-### Release notes {#release-notes}
+## Release notes {#release-notes}
 
 **Primary goal for the reader:** stay informed about new features, improvements, bug fixes, and breaking changes in {{ ydb-short-name }} releases.
 
@@ -85,7 +85,7 @@ Release notes document changes, improvements, and fixes in each new version of {
 
 Release notes are found in the ["Changelog"](../../changelog-server.md) section and help users understand what has changed between versions and decide whether to upgrade.
 
-### Collection of links {#links}
+## Collection of links {#links}
 
 **Primary goal for the reader:** discover additional resources, learning materials, and external content related to {{ ydb-short-name }}.
 

ydb/docs/en/core/contributor/documentation/structure.md

@@ -1,4 +1,4 @@
-## {{ ydb-short-name }} documentation structure
+# {{ ydb-short-name }} documentation structure
 
 This article complements [{#T}](style-guide.md). It explains the current top-level folders of the documentation and what kind of content belongs in each. As a rule of thumb, most top-level sections focus either on a specific target audience (if named "For ...") or on a specific [genre](genres.md).
 

ydb/docs/en/core/dev/system-views.md

@@ -30,7 +30,7 @@ Cumulative fields (`RowReads`, `RowUpdates`, and so on) store the accumulated va
 Table structure:
 
 | Field | Description |
---- | ---
+| --- | --- |
 | `OwnerId` | ID of the SchemeShard serving the table.<br/>Type: `Uint64`.<br/>Key: `0`. |
 | `PathId` | ID of the SchemeShard path.<br/>Type: `Uint64`.<br/>Key: `1`. |
 | `PartIdx` | Partition sequence number.<br/>Type: `Uint64`.<br/>Key: `2`. |
@@ -102,7 +102,7 @@ Query text limit is 4 KB.
 All tables have the same set of fields:
 
 | Field | Description |
---- | ---
+| --- | --- |
 | `IntervalEnd` | The end of a one-minute or one-hour interval.<br/>Type: `Timestamp`.<br/>Key: `0`. |
 | `Rank` | Rank of a top query.<br/>Type: `Uint32`.<br/>Key: `1`. |
 | `QueryText` | Query text.<br/>Type: `Utf8`. |
@@ -181,7 +181,7 @@ Restrictions:
 Table structure:
 
 | Field | Description |
----|---
+| ---|--- |
 | `IntervalEnd` | The end of a one-minute interval.<br/>Type: `Timestamp`.<br/>Key: `0`. |
 | `Rank` | Query rank within an interval (by the SumCPUTime field).<br/>Type: `Uint32`.<br/>Key: `1`. |
 | `QueryText` | Query text.<br/>Type: `Utf8`. |
@@ -253,7 +253,7 @@ These tables contain partitions with peak loads of more than 70% (`CPUCores` > 0
 Both tables have the same set of fields:
 
 | Field | Description |
---- | ---
+| --- | --- |
 | `IntervalEnd` | The end of a one-minute or one-hour interval.<br/>Type: `Timestamp`.<br/>Key: `0`. |
 | `Rank` | Partition rank within an interval (by CPUCores).<br/>Type: `Uint32`.<br/>Key: `1`. |
 | `TabletId` | ID of the tablet serving the partition.<br/>Type: `Uint64`. |