Merge branch 'main' into mbkkt/fix-knn-docs

Author: MBkkt

.github/TESTOWNERS

@@ -13,6 +13,7 @@
 /ydb/core/tx/datashard @ydb-platform/datashard
 /ydb/core/mon_alloc @ydb-platform/datashard
 /ydb/core/tx/coordinator @ydb-platform/datashard
+/ydb/core/statistics @ydb-platform/datashard
 
 #Column Tables Development Team @zverevgeny TEAM:@ydb-platform/cs
 /ydb/core/tx/columnshard @ydb-platform/cs

.github/actions/test_ya/action.yml

@@ -83,12 +83,34 @@ runs:
         mkdir -p $PUBLIC_DIR
 
         echo "LAST_JUNIT_REPORT_XML=$PUBLIC_DIR/last_junit.xml" >> $GITHUB_ENV
-        echo "TESTMO_URL=${{ inputs.testman_url }}" >> $GITHUB_ENV
+        export TESTMO_URL=${{ inputs.testman_url }}
+        echo "TESTMO_URL=$TESTMO_URL" >> $GITHUB_ENV
         echo "SUMMARY_LINKS=$PUBLIC_DIR/summary_links.txt" >> $GITHUB_ENV
         echo "BUILD_PRESET=${{ inputs.build_preset }}" >> $GITHUB_ENV
 
         python3 -m pip install ydb ydb[yc] codeowners
 
+        if [ ${{ inputs.testman_token }} ]; then
+          TESTMO_PROXY_ADDR=127.0.0.1:8888
+          openssl req -x509 -newkey rsa:2048 \
+            -keyout $TMP_DIR/key.pem -out $TMP_DIR/cert.pem \
+            -sha256 -days 1 -nodes -subj "/CN=127.0.0.1"
+          
+          TESTMO_TOKEN=${{ inputs.testman_token }} ./ydb/ci/testmo-proxy/testmo-proxy.py -l $TESTMO_PROXY_ADDR \
+            --cert-file "$TMP_DIR/cert.pem" \
+            --cert-key "$TMP_DIR/key.pem" \
+            --target-timeout 3,60 \
+            --max-request-time 200 \
+            "$TESTMO_URL" > $PUBLIC_DIR/testmo_proxy.log 2>&1 &
+            
+          TESTMO_PROXY_PID=$!
+          echo "TESTMO_PROXY_ADDR=$TESTMO_PROXY_ADDR" >> $GITHUB_ENV
+          echo "TESTMO_PROXY_PID=$TESTMO_PROXY_PID" >> $GITHUB_ENV
+
+          # testmo rejects self-signed cert without this setting
+          echo "NODE_TLS_REJECT_UNAUTHORIZED=0" >> $GITHUB_ENV
+        fi
+
     - name: ya build and test
       id: build
       shell: bash
@@ -260,7 +282,7 @@ runs:
             TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:resources:add-link --name build --url "$TESTMO_RUN_URL" --resources testmo.json
             TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:resources:add-field --name git-sha --type string --value "${GITHUB_SHA:0:7}" --resources testmo.json
             TESTMO_RUN_ID=$(
-              TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:create --instance "$TESTMO_URL" --project-id ${{ inputs.testman_project_id }} \
+              TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:create --instance "https://$TESTMO_PROXY_ADDR" --project-id ${{ inputs.testman_project_id }} \
                 --name "$TESTMO_RUN_NAME" --source "$TESTMO_SOURCE" --resources testmo.json \
                 --tags "$TESTMO_BRANCH_TAG" --tags "$TESTMO_EXTRA_TAG"
             )
@@ -362,7 +384,7 @@ runs:
 
           # upload tests results to YDB
           ydb_upload_run_name="${TESTMO_RUN_NAME// /"_"}"
-          result=`.github/scripts/upload_tests_results.py --test-results-file ${CURRENT_JUNIT_XML_PATH} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull $ydb_upload_run_name --job-name "${{ github.workflow }}" --job-id "${{ github.run_id }}" --branch ${GITHUB_REF_NAME}`
+          result=`.github/scripts/analytics/upload_tests_results.py --test-results-file ${CURRENT_JUNIT_XML_PATH} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull $ydb_upload_run_name --job-name "${{ github.workflow }}" --job-id "${{ github.run_id }}" --branch ${GITHUB_REF_NAME}`
 
           if [ ${{ inputs.testman_token }} ]; then
             # finish testme session
@@ -373,29 +395,12 @@ runs:
             .github/scripts/tests/split-junit.py -o "$TESTMO_JUNIT_REPORT_PARTS" "$CURRENT_JUNIT_XML_PATH"
             # archive unitest reports (transformed)
             tar -C $TESTMO_JUNIT_REPORT_PARTS/.. -czf $PUBLIC_DIR/junit_parts.xml.tar.gz $(basename $TESTMO_JUNIT_REPORT_PARTS)
-
-            TESTMO_PROXY_ADDR=127.0.0.1:8888
-
-            openssl req -x509 -newkey rsa:2048 \
-              -keyout $TMP_DIR/key.pem -out $TMP_DIR/cert.pem \
-              -sha256 -days 1 -nodes -subj "/CN=127.0.0.1"
-            
-            TESTMO_TOKEN=${{ inputs.testman_token }} ./ydb/ci/testmo-proxy/testmo-proxy.py -l $TESTMO_PROXY_ADDR \
-              --cert-file "$TMP_DIR/cert.pem" \
-              --cert-key "$TMP_DIR/key.pem" \
-              --target-timeout 3,10 \
-              --max-request-time 55 \
-              "$TESTMO_URL" &
-            
-            testmo_proxy_pid=$!
 
-            TESTMO_TOKEN=${{ inputs.testman_token }} NODE_TLS_REJECT_UNAUTHORIZED=0 testmo automation:run:submit-thread \
+            TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:submit-thread \
               --instance "https://$TESTMO_PROXY_ADDR" --run-id "$TESTMO_RUN_ID" \
               --results "$TESTMO_JUNIT_REPORT_PARTS/*.xml"
-            
-            kill $testmo_proxy_pid
 
-            TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "$TESTMO_URL" --run-id $TESTMO_RUN_ID
+            TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id $TESTMO_RUN_ID || true
             echo "runid=" >> $GITHUB_OUTPUT
           fi
 
@@ -435,7 +440,10 @@ runs:
       shell: bash
       run: |
         if [ ${{ steps.build.outputs.runid }} ]; then
-          TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "$TESTMO_URL" --run-id ${{ steps.build.outputs.runid }}
+          TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id ${{ steps.build.outputs.runid }} || true
+        fi
+        if [ ${{ inputs.testman_token }} ]; then
+          kill $TESTMO_PROXY_PID
         fi
     - name: analyze tests results
       shell: bash

.github/config/muted_ya.txt

@@ -17,13 +17,16 @@ ydb/core/kqp/ut/query KqpLimits.QueryReplySize
 ydb/core/kqp/ut/query KqpQuery.QueryTimeout
 ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
 ydb/core/kqp/ut/scheme [*/*]*
+ydb/core/kqp/ut/scheme KqpOlapScheme.DropThenAddColumn
 ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
 ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication
 ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter
 ydb/core/kqp/ut/service [*/*]*
 ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
 ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
 ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OlapRWQueries
+ydb/core/kqp/ut/tx KqpSnapshotRead.ReadOnlyTxWithIndexCommitsOnConcurrentWrite+withSink
 ydb/core/persqueue/ut [*/*]*
 ydb/core/persqueue/ut TPQTest.*DirectRead*
 ydb/core/persqueue/ut/ut_with_sdk [*/*]*
@@ -59,7 +62,6 @@ ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/with_offset_ranges_mode_ut [*/
 ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/with_offset_ranges_mode_ut RetryPolicy.TWriteSession_TestBrokenPolicy
 ydb/public/sdk/cpp/client/ydb_topic/ut [*/*]*
 ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.ReadSessionCorrectClose
-ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.WriteRead
 ydb/public/sdk/cpp/client/ydb_topic/ut TSettingsValidation.TestDifferentDedupParams
 ydb/services/datastreams/ut DataStreams.TestGetRecordsStreamWithSingleShard
 ydb/services/datastreams/ut DataStreams.TestPutRecordsOfAnauthorizedUser

.github/scripts/analytics/flaky_tests_history.py

@@ -15,19 +15,17 @@
 config_file_path = f"{dir}/../../config/ydb_qa_db.ini"
 config.read(config_file_path)
 
-build_preset = os.environ.get("build_preset")
-branch = os.environ.get("branch_to_compare")
 
 DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"]
 DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"]
 
 
 def create_tables(pool,  table_path):
-    print(f"> create table: {table_path}")
+    print(f"> create table if not exists:'{table_path}'")
 
     def callee(session):
         session.execute_scheme(f"""
-            CREATE table `{table_path}` (
+            CREATE table IF NOT EXISTS `{table_path}` (
                 `test_name` Utf8 NOT NULL,
                 `suite_folder` Utf8 NOT NULL,
                 `full_name` Utf8 NOT NULL,
@@ -154,18 +152,18 @@ def main():
                     from  `test_results/test_runs_column`
                     where
                         status in ('failure','mute')
-                        and job_name in ('Nightly-run', 'Postcommit_relwithdebinfo')
-                        and build_type = 'relwithdebinfo' and
-                        run_timestamp >= Date('{last_date}') -{history_for_n_day}*Interval("P1D") 
+                        and job_name in ('Nightly-run', 'Postcommit_relwithdebinfo','Postcommit_asan')
+                        and branch = 'main'
+                        and run_timestamp >= Date('{last_date}') -{history_for_n_day}*Interval("P1D") 
                 ) as tests_with_fails
                 cross join (
                     select 
                         DISTINCT DateTime::MakeDate(run_timestamp) as date_base
                     from  `test_results/test_runs_column`
                     where
                         status in ('failure','mute')
-                        and job_name in ('Nightly-run', 'Postcommit_relwithdebinfo')
-                        and build_type = 'relwithdebinfo'
+                        and job_name in ('Nightly-run', 'Postcommit_relwithdebinfo','Postcommit_asan')
+                        and branch = 'main'
                         and run_timestamp>= Date('{last_date}')
                     ) as date_list
                 ) as test_and_date