Merge branch 'main' into dyntables-timestamp-fix

Author: Vadim Averin

.github/CODEOWNERS

@@ -12,5 +12,6 @@
 /ydb/library/yql/providers/s3 @ydb-platform/fq
 /ydb/library/yql/providers/solomon @ydb-platform/fq
 
-/ydb/services/fq/ @ydb-platform/fq
+/ydb/library/yql/yt @Krock21 @Krisha11 @zlobober @gritukan
 
+/ydb/services/fq/ @ydb-platform/fq

.github/actions/build_and_test_ya/action.yml

@@ -30,7 +30,8 @@ inputs:
     default: "small,medium,large"
   test_type:
     type: string
-    default: "unittest,py3test,py2test,pytest"
+    default: ""
+    description: "run only specific test types (or all by default)"
   increment:
     type: boolean
     required: true
@@ -111,6 +112,7 @@ runs:
         build_target: ${{ steps.test_run_choice.outputs.target }}
         build_preset: ${{ inputs.build_preset }}
         test_size: ${{ inputs.test_size }}
+        test_type: ${{ inputs.test_type }}
         testman_token: ${{ fromJSON( inputs.secs ).TESTMO_TOKEN }}
         testman_url: ${{  fromJSON( inputs.vars ).TESTMO_URL }}
         testman_project_id: ${{  fromJSON( inputs.vars ).TESTMO_PROJECT_ID }}

.github/actions/build_ya/action.yml

@@ -37,7 +37,6 @@ runs:
       env:
         build_preset: ${{ inputs.build_preset }}
       run: |
-        echo "SHELLOPTS=xtrace" >> $GITHUB_ENV
         export TMP_DIR=$(pwd)/tmp_build
         echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
         
@@ -54,6 +53,7 @@ runs:
       id: build
       shell: bash
       run: |
+        set -x
         extra_params=()
 
         if [ ! -z "${{ inputs.build_target }}" ]; then
@@ -115,7 +115,7 @@ runs:
         # to be sure
         set -o pipefail
         
-        ./ya make -k --build "${build_type}" --force-build-depends -D'BUILD_LANGUAGES=CPP PY3 PY2 GO' -T --stat -DCONSISTENT_DEBUG \
+        ./ya make -k --build "${build_type}" --force-build-depends -T --stat -DCONSISTENT_DEBUG \
           --log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \
           --cache-size 512G --link-threads "${{ inputs.link_threads }}" \
           "${extra_params[@]}" |& tee $TMP_DIR/ya_make.log && echo "status=true" >> $GITHUB_OUTPUT || (
@@ -128,6 +128,7 @@ runs:
       if: always()
       shell: bash
       run: |
+        set -x
         echo "::group::s3-sync"
         s3cmd sync --acl-private --exclude="ya_make.log" --no-progress --stats --no-check-md5 "$TMP_DIR/" "$S3_BUCKET_PATH/build_logs/"
         s3cmd sync --acl-public --no-progress --stats --no-check-md5 "$TMP_DIR/ya_make.log" "$S3_BUCKET_PATH/build_logs/"
@@ -137,6 +138,7 @@ runs:
       if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
       shell: bash
       run: |
+        set -x
         if [ "${{ steps.build.outputs.status }}" == "failed" ]; then
           echo "Build failed. see the [build logs]($LOG_URL)." | .github/scripts/tests/comment-pr.py --fail
         else

.github/actions/test_ya/action.yml

@@ -7,6 +7,11 @@ inputs:
     required: true
     default: "relwithdebinfo"
     description: "relwithdebinfo, release-asan, release-tsan"
+  test_type:
+    required: false
+    type: string
+    default: ""
+    description: "run only specific test types (or all by default)"
   test_size:
     required: false
     default: "small,medium,large"
@@ -35,7 +40,6 @@ runs:
       id: init
       shell: bash
       run: |
-        echo "SHELLOPTS=xtrace" >> $GITHUB_ENV
         export TMP_DIR=$(pwd)/tmp
         echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
         echo "LOG_DIR=$TMP_DIR/logs" >> $GITHUB_ENV
@@ -54,6 +58,7 @@ runs:
     - name: prepare
       shell: bash
       run: |
+        set -x
         rm -rf $TMP_DIR $JUNIT_REPORT_XML $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
         mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $TEST_ARTIFACTS_DIR $LOG_DIR $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
     
@@ -73,6 +78,7 @@ runs:
       env:
         PR_NUMBER: ${{ github.event.number }}
       run: |
+        set -x
         RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
         BRANCH_TAG="$GITHUB_REF_NAME"
         ARCH="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
@@ -141,11 +147,13 @@ runs:
     - name: ya test
       shell: bash
       run: |
+        set -x
         readarray -d ',' -t test_size < <(printf "%s" "${{ inputs.test_size }}")
+        readarray -d ',' -t test_type < <(printf "%s" "${{ inputs.test_type }}")
 
         params=(
           -T -k
-          ${test_size[@]/#/--test-size=}
+          ${test_size[@]/#/--test-size=} ${test_type[@]/#/--test-type=}
           --cache-size 512G --do-not-output-stderrs
           --stat
           --test-threads "${{ inputs.test_threads }}" --link-threads "${{ inputs.link_threads }}"
@@ -214,6 +222,7 @@ runs:
     - name: postprocess junit report
       shell: bash
       run: |
+        set -x
         .github/scripts/tests/transform-ya-junit.py -i \
           -m .github/config/muted_ya.txt \
           --ya-out "$OUT_DIR" \
@@ -234,6 +243,7 @@ runs:
       if: inputs.testman_token
       shell: bash
       run: |
+        set -x
         PROXY_ADDR=127.0.0.1:8888
 
         openssl req -x509 -newkey rsa:2048 \
@@ -265,6 +275,7 @@ runs:
       shell: bash
       if: always()
       run: |
+        set -x
         mkdir $ARTIFACTS_DIR/summary/
         
         cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
@@ -280,6 +291,7 @@ runs:
       if: always()
       shell: bash
       run: |
+        set -x
         echo "::group::s3-sync"
         s3cmd sync -r --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$ARTIFACTS_DIR/" "$S3_BUCKET_PATH/"
         s3cmd sync -r --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$TEST_ARTIFACTS_DIR/" "$S3_TEST_ARTIFACTS_BUCKET_PATH/"
@@ -289,6 +301,7 @@ runs:
       if: always()
       shell: bash
       run: |
+        set -x
         echo "::group::s3-sync"
         s3cmd sync --follow-symlinks --acl-private --no-progress --stats --no-check-md5 "$LOG_DIR/" "$S3_BUCKET_PATH/test_logs/"
         echo "::endgroup::"

.github/config/muted_ya.txt

@@ -6,17 +6,21 @@ ydb/core/client/ut TFlatTest.AutoSplitMergeQueue
 ydb/core/cms/ut_sentinel TSentinelTests.BSControllerCantChangeStatus
 ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
 ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
-ydb/core/kqp/ut/federated_query/generic *
-ydb/core/kqp/ut/olap KqpOlap.IndexesActualization
-ydb/core/kqp/ut/olap KqpOlap.BlobsSharing*
+ydb/core/kqp/ut/olap KqpOlapIndexes.IndexesActualization
+ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
 ydb/core/kqp/ut/olap KqpOlap.ScanQueryOltpAndOlap
-ydb/core/kqp/ut/olap KqpOlap.StatsUsageWithTTL
+ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
 ydb/core/kqp/ut/olap KqpOlap.YqlScriptOltpAndOlap
+ydb/core/kqp/ut/service KqpQueryService.TableSink_ReplaceFromSelectLargeOlap
 ydb/core/kqp/ut/pg KqpPg.CreateIndex
 ydb/core/kqp/ut/query KqpLimits.QueryReplySize
 ydb/core/kqp/ut/query KqpQuery.QueryTimeout
 ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
 ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
+ydb/core/kqp/ut/scheme KqpOlap.OlapRead_GenericQuerys
+ydb/core/kqp/ut/scheme KqpOlap.OlapRead_StreamGenericQuery
+ydb/core/kqp/ut/scheme KqpOlap.OlapRead_UsesGenericQueryOnJoinWithDataShardTable
+ydb/core/kqp/ut/scheme KqpQueryService.TableSink_ReplaceFromSelectLargeOlap
 ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter
 ydb/core/kqp/ut/scheme [44/50]*
 ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
@@ -35,7 +39,7 @@ ydb/core/viewer/ut Viewer.TabletMerging
 ydb/core/viewer/ut Viewer.TabletMergingPacked
 ydb/library/actors/http/ut HttpProxy.TooLongHeader
 ydb/library/actors/http/ut sole*
-ydb/library/yql/providers/generic/connector/tests *
+ydb/library/yql/providers/generic/connector/tests/datasource/ydb* *
 ydb/public/lib/ydb_cli/topic/ut TTopicReaderTests.TestRun_ReadOneMessage
 ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut RetryPolicy.RetryWithBatching
 ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut RetryPolicy.TWriteSession_TestBrokenPolicy
@@ -54,7 +58,6 @@ ydb/services/keyvalue/ut KeyValueGRPCService.SimpleCopyUnexistedKey
 ydb/services/keyvalue/ut sole*
 ydb/services/persqueue_v1/ut TPersQueueTest.CheckACLForGrpcWrite
 ydb/services/persqueue_v1/ut TPersQueueTest.SetupLockSession
-ydb/services/persqueue_v1/ut TPersQueueTest.TopicServiceCommitOffsetBadOffsets
 ydb/services/persqueue_v1/ut TPQCompatTest.BadTopics
 ydb/services/persqueue_v1/ut TPersQueueTest.DirectRead*Cache*
 ydb/services/persqueue_v1/ut [3/10]*
@@ -67,23 +70,18 @@ ydb/services/ydb/ut YdbOlapStore.LogPagingAfter-NotNull
 ydb/services/ydb/ut YdbQueryService.TestCreateAndAttachSession
 ydb/services/ydb/ut YdbScripting.BasicV0
 ydb/services/ydb/ut YdbYqlClient.TestYqlSessionClosed
-ydb/tests/fq/generic *
 ydb/tests/fq/mem_alloc *
 ydb/tests/fq/multi_plane *
 ydb/tests/fq/plans test_stats_mode.py.TestStatsMode.test_mode[v1-STATS_MODE_FULL-client0]
 ydb/tests/fq/yds *
 ydb/tests/functional/audit *
 ydb/tests/functional/blobstorage test_replication.py.TestReplicationAfterNodesRestart.test_replication*
-ydb/tests/functional/clickbench test.py.test_plans*
-ydb/tests/functional/clickbench test.py.test_run*
 ydb/tests/functional/kqp/kqp_indexes ConsistentIndexRead.InteractiveTx
 ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach
 ydb/tests/functional/restarts test_restarts.py.*
 ydb/tests/functional/serverless test_serverless.py.test_database_with_disk_quotas[enable_alter_database_create_hive_first--*]
 ydb/tests/functional/tenants test_dynamic_tenants.py.*
 ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
 ydb/tests/functional/tenants test_tenants.py.*
-ydb/tests/functional/ydb_cli test_ydb_scripting.py.TestScriptingServiceHelp.test_help
-ydb/tests/functional/ydb_cli test_ydb_scripting.py.TestScriptingServiceHelp.test_help_ex
 ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout
 ydb/tests/functional/rename [test_rename.py */10] chunk chunk

.github/scripts/tests/generate-summary.py

@@ -23,6 +23,10 @@ class TestStatus(Enum):
     SKIP = 3
     MUTE = 4
 
+    @property
+    def is_error(self):
+        return self in (TestStatus.FAIL, TestStatus.ERROR, TestStatus.MUTE)
+
     def __lt__(self, other):
         return self.value < other.value
 
@@ -79,6 +83,7 @@ def from_junit(cls, testcase):
         log_urls = {
             'Log': get_property_value(testcase, "url:Log"),
             'log': get_property_value(testcase, "url:log"),
+            'logsdir': get_property_value(testcase, "url:logsdir"),
             'stdout': get_property_value(testcase, "url:stdout"),
             'stderr': get_property_value(testcase, "url:stderr"),
         }

.github/scripts/tests/templates/summary.html

@@ -4,7 +4,7 @@
         th {
             text-transform: uppercase;
         }
-        
+
         th, td {
             padding: 5px;
         }
@@ -28,12 +28,56 @@
         span.test_mute {
             color: blue;
         }
+        button.copy {
+            float: right;
+        }
         table > tbody > tr > td:nth-child(2),
         table > tbody > tr > td:nth-child(3),
         table > tbody > tr > td:nth-child(4) {
             text-align: center;
         }
     </style>
+    <script>
+        function copyTestNameToClipboard(text) {
+            const full_name = text.trim();
+            const pieces = /(.+)\/([^$]+)$/.exec(full_name);
+
+            if (!pieces) {
+                console.error("Unable to split path/test for %o", full_name);
+                return;
+            }
+            let [path, testName] = [pieces[1], pieces[2]];
+
+            const namePieces = testName.split('.');
+
+            if (namePieces.length === 2) {
+                testName = namePieces[0] + '::' + namePieces[1];
+            } else {
+                testName = namePieces[0] + '.' + namePieces[1] + '::' + namePieces.slice(2).join('::');
+            }
+
+            const cmdArg = `${path} -F '${testName}'`;
+
+            console.log(cmdArg);
+
+            navigator.clipboard.writeText(cmdArg).catch(
+                () => {
+                    console.error("Unable to copy %o to clipboard", cmdArg);
+                }
+            );
+        }
+        function copyOnPress(event) {
+            event.preventDefault();
+            copyTestNameToClipboard(event.target.previousElementSibling.textContent)
+        }
+
+        document.addEventListener("DOMContentLoaded", function() {
+            const els = document.getElementsByClassName("copy");
+            for (let i = 0; i < els.length; i++) {
+                els[i].addEventListener('click', copyOnPress);
+            }
+        });
+    </script>
 </head>
 <body>
 {% for status in status_order %}
@@ -52,7 +96,10 @@ <h1 id="{{ status.name}}">{{ status.name }} ({{ tests[status] | length }})</h1>
     <tbody>
     {% for t in tests[status] %}
     <tr>
-        <td>{{ t.full_name }}</td>
+        <td>
+            <span>{{ t.full_name }}</span>
+            {% if status.is_error %}<button class="copy" title="Copy test filter to clipboard">Copy test filter</button>{% endif %}
+        </td>
         <td><span title="{{ t.elapsed }}s">{{ t.elapsed_display }}</span></td>
         <td>
             <span class="test_status test_{{ t.status_display }}">{{ t.status_display }}</span>

.github/scripts/tests/transform-ya-junit.py

@@ -6,6 +6,7 @@
 import sys
 import urllib.parse
 import zipfile
+from typing import Set
 from xml.etree import ElementTree as ET
 from mute_utils import mute_target, pattern_to_re
 from junit_utils import add_junit_link_property, is_faulty_testcase
@@ -54,7 +55,7 @@ class YTestReportTrace:
     def __init__(self, out_root):
         self.out_root = out_root
         self.traces = {}
-        self.logs_dir = None
+        self.logs_dir = set()
 
     def abs_path(self, path):
         return path.replace("$(BUILD_ROOT)", self.out_root)
@@ -83,8 +84,7 @@ def load(self, subdir):
                         cls = cls.replace("::", ".")
                         self.traces[(cls, subtest)] = event
                         logs_dir = self.abs_path(event['logs']['logsdir'])
-                        self.logs_dir = logs_dir
-            break
+                        self.logs_dir.add(logs_dir)
 
     def has(self, cls, name):
         return (cls, name) in self.traces
@@ -144,18 +144,21 @@ def save_log(build_root, fn, out_dir, log_url_prefix, trunc_size):
     return f"{log_url_prefix}{quoted_fpath}"
 
 
-def save_zip(suite_name, out_dir, url_prefix, logs_dir):
+def save_zip(suite_name, out_dir, url_prefix, logs_dir: Set[str]):
     arc_name = f"{suite_name.replace('/', '-')}.zip"
 
     arc_fn = os.path.join(out_dir, arc_name)
 
     zf = zipfile.ZipFile(arc_fn, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=9)
 
-    log_print(f"put {logs_dir} into {arc_name}")
-    for root, dirs, files in os.walk(logs_dir):
-        for f in files:
-            filename = os.path.join(root, f)
-            zf.write(filename, os.path.relpath(filename, logs_dir))
+    for path in logs_dir:
+        # path is .../test-results/black/testing_out_stuff
+        log_print(f"put {path} into {arc_name}")
+        test_type = os.path.basename(os.path.dirname(path))
+        for root, dirs, files in os.walk(path):
+            for f in files:
+                filename = os.path.join(root, f)
+                zf.write(filename, os.path.join(test_type, os.path.relpath(filename, path)))
     zf.close()
 
     quoted_fpath = urllib.parse.quote(arc_name)