Merge branch 'main' into dstool/improve_balance_command

Author: kruall

.github/CODEOWNERS

@@ -9,6 +9,7 @@
 /tools @ydb-platform/ci
 /util @ydb-platform/ci
 /vendor @ydb-platform/ci
+/yql @ydb-platform/ci
 /yt @ydb-platform/ci
 
 /ydb/core/fq/ @ydb-platform/fq
@@ -18,8 +19,10 @@
 
 /ydb/library/yql/ @ydb-platform/yql
 /ydb/library/yql/dq @ydb-platform/yql @ydb-platform/qp
+/ydb/library/yql/dq/actors/common @ydb-platform/fq
 /ydb/library/yql/providers/common/http_gateway @ydb-platform/fq
 /ydb/library/yql/providers/common/db_id_async_resolver @ydb-platform/fq
+/ydb/library/yql/providers/common/pushdown @ydb-platform/fq
 /ydb/library/yql/providers/generic @ydb-platform/fq
 /ydb/library/yql/providers/pq @ydb-platform/fq
 /ydb/library/yql/providers/s3 @ydb-platform/fq

.github/config/muted_ya.txt

@@ -1,15 +1,15 @@
 ydb/core/blobstorage/dsproxy/ut TBlobStorageProxySequenceTest.TestBlock42PutWithChangingSlowDisk
 ydb/core/blobstorage/dsproxy/ut_fat TBlobStorageProxyTest.TestBatchedPutRequestDoesNotContainAHugeBlob
-ydb/core/blobstorage/pdisk/ut TPDiskTest.TestMultipleLogSpliceNonceJump
+ydb/core/blobstorage/pdisk/ut TPDiskTest.AllRequestsAreAnsweredOnPDiskRestart
+ydb/core/blobstorage/ut_blobstorage/ut_huge HugeBlobOnlineSizeChange.Compaction
+ydb/core/blobstorage/ut_blobstorage/ut_huge [*/*] chunk chunk
 ydb/core/blobstorage/ut_blobstorage/ut_scrub BlobScrubbing.block42
 ydb/core/blobstorage/ut_blobstorage/ut_scrub BlobScrubbing.mirror3dc
 ydb/core/blobstorage/ut_blobstorage/ut_scrub BlobScrubbing.mirror3of4
 ydb/core/blobstorage/ut_mirror3of4 Mirror3of4.ReplicationSmall
 ydb/core/blobstorage/ut_vdisk TBsVDiskGC.GCPutKeepBarrierSync
 ydb/core/blobstorage/ut_vdisk TBsVDiskManyPutGet.ManyPutRangeGetCompactionIndexOnly
-ydb/core/client/ut TClientTest.PromoteFollower
-ydb/core/client/ut TClientTest.ReadFromFollower
-ydb/core/client/ut TFlatTest.AutoSplitMergeQueue
+ydb/core/blobstorage/ut_vdisk [*/*] chunk chunk
 ydb/core/cms/ut_sentinel_unstable TSentinelUnstableTests.BSControllerCantChangeStatus
 ydb/core/cms/ut_sentinel_unstable sole chunk chunk
 ydb/core/cms/ut_sentinel_unstable sole+chunk+chunk
@@ -22,12 +22,27 @@ ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertEvWrite
 ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertViaLegacyScripting-Streaming
 ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
 ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.BlobsSharingSplit1_1
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.BlobsSharingSplit1_1_clean
 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.BlobsSharingSplit1_1_clean_with_restarts
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.BlobsSharingSplit1_3_1
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.BlobsSharingSplit1_3_2_1_clean
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.BlobsSharingSplit3_1
 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.ChangeSchemaAndSplit
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.HugeSchemeHistory
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.MultipleMerge
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.MultipleMergesWithRestartsAfterWait
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.MultipleMergesWithRestartsWhenWait
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.MultipleSchemaVersions
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.MultipleSplits
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.MultipleSplitsThenMerges
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.MultipleSplitsWithRestartsAfterWait
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.MultipleSplitsWithRestartsWhenWait
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.SplitEmpty
 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.TableReshardingConsistency64
 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.TableReshardingModuloN
 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.UpsertWhileSplitTest
-ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
+ydb/core/kqp/ut/olap KqpOlapTiering.Eviction
 ydb/core/kqp/ut/olap KqpOlapWrite.TierDraftsGCWithRestart
 ydb/core/kqp/ut/olap [*/*] chunk chunk
 ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable+ColumnStore
@@ -41,9 +56,11 @@ ydb/core/kqp/ut/scheme [*/*]+chunk+chunk
 ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
 ydb/core/kqp/ut/service [*/*] chunk chunk
 ydb/core/kqp/ut/service [*/*]+chunk+chunk
-ydb/core/kqp/ut/sysview KqpSystemView.PartitionStatsFollower
+ydb/core/kqp/ut/spilling KqpScanSpilling.SelfJoinQueryService
 ydb/core/mind/hive/ut THiveTest.DrainWithHiveRestart
+ydb/core/persqueue/ut TPQTest.TestReadAndDeleteConsumer
 ydb/core/persqueue/ut [*/*] chunk chunk
+ydb/core/persqueue/ut/ut_with_sdk TopicAutoscaling.CDC_Write
 ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
 ydb/core/tx/schemeshard/ut_move_reboots TSchemeShardMoveRebootsTest.WithData
 ydb/core/tx/schemeshard/ut_move_reboots TSchemeShardMoveRebootsTest.WithDataAndPersistentPartitionStats
@@ -53,40 +70,32 @@ ydb/core/tx/tiering/ut ColumnShardTiers.TTLUsage
 ydb/core/tx/tx_proxy/ut_ext_tenant TExtSubDomainTest.CreateTableInsideAndAlterDomainAndTable-AlterDatabaseCreateHiveFirst-false
 ydb/core/viewer/ut Viewer.TabletMerging
 ydb/core/viewer/ut Viewer.TabletMergingPacked
+ydb/core/viewer/ut Viewer.TenantInfo5kkTablets
 ydb/library/actors/http/ut HttpProxy.TooLongHeader
 ydb/library/actors/http/ut sole chunk chunk
 ydb/library/actors/http/ut sole+chunk+chunk
 ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll
 ydb/library/actors/interconnect/ut_huge_cluster sole chunk chunk
+ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse test.py.test_select_positive[primitive_types_non_nullable_NATIVE-dqrun]
 ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server test.py.test_select_positive[constant-dqrun]
-ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server test.py.test_select_positive[count_rows-dqrun]
 ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server test.py.test_select_positive[primitives-dqrun]
 ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server test.py.test_select_positive[primitives-kqprun]
-ydb/library/yql/providers/generic/connector/tests/datasource/mysql test.py.test_select_positive[constant-dqrun]
 ydb/library/yql/providers/generic/connector/tests/datasource/mysql test.py.test_select_positive[constant-kqprun]
-ydb/library/yql/providers/generic/connector/tests/datasource/mysql test.py.test_select_positive[count_rows-dqrun]
-ydb/library/yql/providers/generic/connector/tests/datasource/mysql test.py.test_select_positive[count_rows-kqprun]
 ydb/library/yql/providers/generic/connector/tests/datasource/mysql test.py.test_select_positive[primitives-dqrun]
 ydb/library/yql/providers/generic/connector/tests/datasource/mysql test.py.test_select_positive[primitives-kqprun]
-ydb/library/yql/providers/generic/connector/tests/datasource/mysql test.py.test_select_positive[pushdown0-dqrun]
-ydb/library/yql/providers/generic/connector/tests/datasource/mysql test.py.test_select_positive[pushdown0-kqprun]
 ydb/library/yql/providers/generic/connector/tests/datasource/oracle test.py.test_select_positive[PRIMITIVES-dqrun]
+ydb/library/yql/providers/generic/connector/tests/datasource/postgresql test.py.test_select_positive[primitive_types-dqrun]
 ydb/library/yql/providers/generic/connector/tests/datasource/ydb test.py.test_select_positive[column_selection_asterisk-dqrun]
-ydb/library/yql/providers/generic/connector/tests/datasource/ydb test.py.test_select_positive[pushdown-kqprun]
+ydb/library/yql/providers/generic/connector/tests/datasource/ydb test.py.test_select_positive[primitive_types-kqprun]
 ydb/library/yql/providers/generic/connector/tests/join test.py.test_join[join_ch_ch-dqrun]
 ydb/library/yql/tests/sql/hybrid_file/part1 test.py.test[in-in_noansi_join--Debug]
-ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/with_offset_ranges_mode_ut [*/*] chunk chunk
-ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/with_offset_ranges_mode_ut [*/*]+chunk+chunk
 ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.ReadSessionCorrectClose
 ydb/public/sdk/cpp/client/ydb_topic/ut [*/*] chunk chunk
 ydb/public/sdk/cpp/client/ydb_topic/ut [*/*]+chunk+chunk
-ydb/services/datastreams/ut DataStreams.TestGetRecordsStreamWithSingleShard
 ydb/services/datastreams/ut DataStreams.TestPutRecordsCornerCases
 ydb/services/datastreams/ut DataStreams.TestReservedConsumersMetering
 ydb/services/datastreams/ut DataStreams.TestReservedStorageMetering
 ydb/services/keyvalue/ut KeyValueGRPCService.SimpleConcatUnexistedKey
-ydb/services/keyvalue/ut KeyValueGRPCService.SimpleExecuteTransactionWithWrongGeneration
-ydb/services/keyvalue/ut KeyValueGRPCService.SimpleGetStorageChannelStatus
 ydb/services/keyvalue/ut sole chunk chunk
 ydb/services/keyvalue/ut sole+chunk+chunk
 ydb/services/persqueue_v1/ut TPersQueueTest.DirectReadCleanCache
@@ -119,49 +128,29 @@ ydb/tests/fq/multi_plane [test_dispatch.py]+chunk+chunk
 ydb/tests/fq/multi_plane [test_retry.py] chunk chunk
 ydb/tests/fq/multi_plane [test_retry.py]+chunk+chunk
 ydb/tests/fq/multi_plane [test_retry_high_rate.py] chunk chunk
-ydb/tests/fq/multi_plane test_dispatch.py.TestMapping.test_mapping
 ydb/tests/fq/multi_plane test_retry.py.TestRetry.test_low_rate[kikimr0]
 ydb/tests/fq/s3 [test_format_setting.py] chunk chunk
 ydb/tests/fq/yds [*/*] chunk chunk
 ydb/tests/fq/yds [*/*]+chunk+chunk
 ydb/tests/fq/yds test_2_selects_limit.py.TestSelectLimit.test_select_same[v1]
 ydb/tests/fq/yds test_2_selects_limit.py.TestSelectLimit.test_select_sequence[v1]
 ydb/tests/fq/yds test_big_state.py.TestBigState.test_gt_8mb[v1]
-ydb/tests/fq/yds test_kill_pq_bill.py.TestKillPqBill.test_do_not_bill_pq[v1-mvp_external_ydb_endpoint0]
 ydb/tests/fq/yds test_mem_alloc.py.TestMemAlloc.test_hop_alloc[v1]
 ydb/tests/fq/yds test_mem_alloc.py.TestMemAlloc.test_join_alloc[v1]
-ydb/tests/fq/yds test_pq_read_write.py.TestPqReadWrite.test_pq_read_write[v1-with_checkpoints-mvp_external_ydb_endpoint0]
-ydb/tests/fq/yds test_pq_read_write.py.TestPqReadWrite.test_pq_read_write[v1-without_checkpoints-mvp_external_ydb_endpoint0]
 ydb/tests/fq/yds test_recovery.py.TestRecovery.test_ic_disconnection
-ydb/tests/fq/yds test_recovery.py.TestRecovery.test_program_state_recovery
-ydb/tests/fq/yds test_recovery.py.TestRecovery.test_program_state_recovery_error_if_no_states
-ydb/tests/fq/yds test_row_dispatcher.py.TestPqRowDispatcher.test_3_sessions
-ydb/tests/fq/yds test_row_dispatcher.py.TestPqRowDispatcher.test_filter
-ydb/tests/fq/yds test_row_dispatcher.py.TestPqRowDispatcher.test_read_raw_format_with_row_dispatcher
 ydb/tests/fq/yds test_row_dispatcher.py.TestPqRowDispatcher.test_restart_compute_node
-ydb/tests/fq/yds test_row_dispatcher.py.TestPqRowDispatcher.test_scheme_error
 ydb/tests/fq/yds test_row_dispatcher.py.TestPqRowDispatcher.test_start_new_query
-ydb/tests/fq/yds test_row_dispatcher.py.TestPqRowDispatcher.test_stop_start
-ydb/tests/fq/yds test_row_dispatcher.py.TestPqRowDispatcher.test_stop_start_with_filter
 ydb/tests/fq/yds test_select_limit_db_id.py.TestSelectLimitWithDbId.test_select_same_with_id[v1-mvp_external_ydb_endpoint0]
-ydb/tests/fq/yds test_watermarks.py.TestWatermarks.test_idle_watermarks[v1-mvp_external_ydb_endpoint0]
-ydb/tests/fq/yds test_watermarks.py.TestWatermarks.test_pq_watermarks[v1-mvp_external_ydb_endpoint0]
 ydb/tests/fq/yds test_yds_bindings.py.TestBindings.test_yds_insert[v1]
-ydb/tests/fq/yds test_yq_streaming.py.TestYqStreaming.test_match_recognize_sink[v1]
-ydb/tests/fq/yds test_yq_streaming.py.TestYqStreaming.test_state_load_mode[v1]
 ydb/tests/fq/yt/kqp_yt_file/part18 test.py.test[pg-join_brackets2-default.txt]
-ydb/tests/functional/audit test_auditlog.py.test_cloud_ids_are_logged[attrs0]
-ydb/tests/functional/audit test_auditlog.py.test_cloud_ids_are_logged[attrs1]
 ydb/tests/functional/audit test_auditlog.py.test_dml_requests_arent_logged_when_sid_is_expected
-ydb/tests/functional/audit test_auditlog.py.test_single_dml_query_logged[delete]
 ydb/tests/functional/hive test_drain.py.TestHive.test_drain_on_stop
 ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach
 ydb/tests/functional/rename [test_rename.py */*] chunk chunk
 ydb/tests/functional/restarts test_restarts.py.TestRestartSingleBlock42.test_restart_single_node_is_ok
 ydb/tests/functional/serializable test.py.test_local
 ydb/tests/functional/serverless test_serverless.py.test_database_with_disk_quotas[enable_alter_database_create_hive_first--false]
 ydb/tests/functional/serverless test_serverless.py.test_database_with_disk_quotas[enable_alter_database_create_hive_first--true]
-ydb/tests/functional/suite_tests test_postgres.py.TestPGSQL.test_sql_suite[plan-jointest/join2.test]
 ydb/tests/functional/tenants test_dynamic_tenants.py.test_create_and_drop_tenants[enable_alter_database_create_hive_first--false]
 ydb/tests/functional/tenants test_dynamic_tenants.py.test_create_and_drop_tenants[enable_alter_database_create_hive_first--true]
 ydb/tests/functional/tenants test_dynamic_tenants.py.test_create_and_drop_the_same_tenant2[enable_alter_database_create_hive_first--false]
@@ -172,7 +161,6 @@ ydb/tests/functional/tenants test_tenants.py.TestTenants.test_list_database_abov
 ydb/tests/functional/tenants test_tenants.py.TestTenants.test_list_database_above[enable_alter_database_create_hive_first--true]
 ydb/tests/functional/tenants test_tenants.py.TestTenants.test_stop_start[enable_alter_database_create_hive_first--false]
 ydb/tests/functional/tenants test_tenants.py.TestTenants.test_stop_start[enable_alter_database_create_hive_first--true]
-ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset[tsv-additional_args2-column]
 ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[Test64BitErrorChecking]
 ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestArrayValueBackend]
 ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestBinaryByteSliceToInt]

.github/workflows/prewarm-debug.yaml

@@ -1,7 +1,7 @@
 name: Prewarm-debug
 on:
   schedule:
-    - cron: "0 */2 * * *"  # At minute 0 past every 2nd hour
+    - cron: "0 * * * *"  # At minute 0
   workflow_dispatch:
 jobs:
   build_and_test:

BUILD.md

@@ -71,7 +71,7 @@ git checkout CLI_2.5.0
 
 You can use the native Yatool build for x86_64 under Ubuntu Linux 18+.
 
-Run `ya make` from the repository root, followed by an optional relative path to the build target. By default, `ya make` compiles a `debug` configuration, while you can choose to have `release` or `relwithdebinfo`. The latter is recommended for faster builds, as is leverages the YDB remote build cache.
+Run `ya make` from the repository root, followed by an optional relative path to the build target. By default, `ya make` compiles a `debug` configuration, while you can choose to have `release` or `relwithdebinfo`. The latter is recommended for faster builds, as it leverages the YDB remote build cache.
 
 To build a binary, Yatool downloads and caches relevant toolchains from the YDB S3 storage, so you do not need to install anything on the build machine.
 
@@ -97,7 +97,7 @@ You can use CMake to build YDB for various architectures and operating systems.
 
 YDB server can be built for Ubuntu 18.04, 20.04 and 22.04. Other Linux distributions are likely to work, but additional effort may be required.
 
-YDB CLI can be built for Ubuntu 18+, Windows, and MacOS. Instructions below are provided for Ubuntu only, other options are to be described later.
+YDB CLI can be built for Ubuntu 18+, Windows, and macOS. Instructions below are provided for Ubuntu only, other options are to be described later.
 
 ### For Ubuntu versions earlier than 22.04
 
@@ -133,7 +133,9 @@ It is recommended to build YDB on Ubuntu 22.04. Follow these additional instruct
 ### Install dependencies
 
 ```bash
-sudo apt-get -y install git cmake python3-pip ninja-build antlr3 m4 clang-14 lld-14 libidn11-dev libaio1 libaio-dev llvm-14
+sudo apt-get update && \
+sudo apt-get -y install git cmake python3-pip ninja-build antlr3 m4 clang-14 lld-14 libidn11-dev \
+ libaio1 libaio-dev llvm-14 curl
 sudo pip3 install conan==1.59 grpcio-tools==1.57.0
 
 ```

build/conf/autoincludes.json

@@ -0,0 +1,5 @@
+[
+    "library/cpp/geo",
+    "util"
+]
+

build/conf/compilers/gnu_compiler.conf

@@ -68,6 +68,16 @@ when ($CLANG16 == "yes") {
     CFLAGS+=-Wno-array-parameter -Wno-deprecate-lax-vec-conv-all -Wno-unqualified-std-cast-call -Wno-unused-but-set-parameter -Wno-implicit-function-declaration -Wno-int-conversion -Wno-incompatible-function-pointer-types -Wno-address-of-packed-member
 }
 
+when ($CLANG18 == "yes") {
+    CFLAGS+=-Wno-array-parameter -Wno-deprecate-lax-vec-conv-all -Wno-unqualified-std-cast-call -Wno-unused-but-set-parameter -Wno-implicit-function-declaration -Wno-int-conversion -Wno-incompatible-function-pointer-types -Wno-address-of-packed-member
+    CFLAGS+=-Wno-deprecated-this-capture -Wno-c++11-narrowing-const-reference -Wno-missing-field-initializers \
+            -Wno-packed-non-pod -Wno-format -Wno-vla-cxx-extension -Wno-invalid-offsetof -Wno-exceptions \
+            -Wno-include-angled-in-module-purview -Wno-unused-parameter
+    when ($MAPSMOBI_BUILD_TARGET == "yes") {
+        CFLAGS+=-Wno-deprecated-declarations
+    }
+}
+
 when ($MSAN_TRACK_ORIGIN == "yes") {
     CFLAGS+=-fsanitize-memory-track-origins=2
 }
@@ -216,7 +226,7 @@ _C_CPP_KV_STYLE_P=p CC
 _C_CPP_KV_STYLE_PC=pc green
 
 _C_CPP_WRAPPER=$YMAKE_PYTHON3 ${input:"build/scripts/wrapcc.py"} ${input:SRC}
-when ($CLANG_TIDY_ARGS || $RETRY_ARGS || $YNDEXER_ARGS || $OPENSOURCE == "yes") {
+when ($CLANG_TIDY_ARGS || $RETRY_ARGS || $YNDEXER_ARGS || $OPENSOURCE == "yes" || $RAW_COMPILE_CPP_CMD == "yes" ) {
     _C_CPP_WRAPPER=
 }
 

build/conf/go.conf

@@ -817,7 +817,6 @@ module GO_LIBRARY: _GO_BASE_UNIT {
     SET(MODULE_TYPE LIBRARY)
     SET(PEERDIR_TAGS GO GO_PROTO GO_FBS __EMPTY__ RESOURCE_LIB PACKAGE_UNION)
 
-    _MAKEFILE_INCLUDE_LIKE_DEPS+=.yo.snapshot.json
     MODULE_SUFFIX=.a
 }