main2 -> graph_v2

.github/CODEOWNERS

@@ -15,3 +15,9 @@
 /ydb/library/yql/yt @Krock21 @Krisha11 @zlobober @gritukan
 
 /ydb/services/fq/ @ydb-platform/fq
+
+/ydb/core/kafka_proxy @ydb-platform/Topics
+/ydb/core/persqueue @ydb-platform/Topics
+/ydb/services/datastreams @ydb-platform/Topics
+/ydb/services/deprecated/persqueue_v0 @ydb-platform/Topics
+/ydb/services/persqueue_v1 @ydb-platform/Topics

.github/actions/build_analytics/action.yml

@@ -0,0 +1,54 @@
+name: build_analytics
+description: Gather and send build analytics 
+inputs:
+  build_preset:
+    type: string
+    default: ""
+  build_target:
+    type: string
+    default: ""
+  secs:
+    type: string
+    default: ""
+  vars:
+    type: string
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Prepare s3cmd
+      uses: ./.github/actions/s3cmd
+      with:
+        s3_bucket: ${{ fromJSON( inputs.vars ).AWS_BUCKET }}
+        s3_endpoint: ${{ fromJSON( inputs.vars ).AWS_ENDPOINT }}
+        s3_key_id: ${{ fromJSON( inputs.secs ).AWS_KEY_ID }}
+        s3_key_secret: ${{ fromJSON( inputs.secs ).AWS_KEY_VALUE }}
+        folder_prefix: ya-
+        build_preset: ${{ inputs.build_preset }}
+
+    - name: Run analytics tools
+      shell: bash
+      run: |
+        set -ex
+        # FIXME: target name may be not the same as dir name
+        export TARGET_NAME=`basename ${{ inputs.build_target }}`
+        export TARGET_DIR=${{ inputs.build_target }}
+        export build_preset="${{ inputs.build_preset }}" 
+        export build_target="${{ inputs.build_target }}" 
+        ./ya tool bloat --linker-map $TARGET_DIR/$TARGET_NAME.map.lld --input $TARGET_DIR/$TARGET_NAME --save-html ya_bloat_html
+        ./ydb/ci/build_bloat/main.py --build-dir . --html-dir-cpp html_cpp_impact --html-dir-headers html_headers_impact
+        ./ydb/ci/build_bloat/ydb_upload.py --html-dir-cpp html_cpp_impact --html-dir-headers html_headers_impact
+
+    - name: Upload results
+      shell: bash
+      run: |
+        set -ex
+        s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "ya_bloat_html/" "$S3_BUCKET_PATH/ya_bloat_html/"
+        echo "[ya bloat tool]($S3_URL_PREFIX/ya_bloat_html/index.html) (better use Safari, because it is too large for Chrome)" >> $GITHUB_STEP_SUMMARY
+
+        s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_cpp_impact/" "$S3_BUCKET_PATH/html_cpp_impact/"
+        echo "[cpp compilation time]($S3_URL_PREFIX/html_cpp_impact/index.html)" >> $GITHUB_STEP_SUMMARY
+
+        s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_headers_impact/" "$S3_BUCKET_PATH/html_headers_impact/"
+        echo "[headers impact]($S3_URL_PREFIX/html_headers_impact/index.html)" >> $GITHUB_STEP_SUMMARY
+

.github/actions/build_and_test_ya/action.yml

@@ -42,6 +42,9 @@ inputs:
   put_build_results_to_cache:
     type: boolean
     default: true
+  additional_ya_make_args:
+    type: string
+    default: ""
   secs:
     type: string
     default: ""
@@ -66,7 +69,17 @@ runs:
         check_url=$(curl -s $jobs_url | jq --arg n "$BUILD_PRESET" -r '.jobs[] | select(.name | contains($n)) | .html_url')
 
         echo "Pre-commit [check]($check_url) for $(git rev-parse HEAD) has started." | .github/scripts/tests/comment-pr.py --rewrite
-
+
+        curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+          -d '{"state":"pending","description":"The check has been started","context":"build_${{inputs.build_preset}}"}'
+
+        if [[ "${{inputs.run_tests}}" == "true" ]];then
+          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+            -d '{"state":"pending","description":"The check has been started","context":"test_${{inputs.build_preset}}"}'
+        fi
+
     - name: Prepare s3cmd
       uses: ./.github/actions/s3cmd
       with:
@@ -77,6 +90,20 @@ runs:
         folder_prefix: ya-
         build_preset: ${{ inputs.build_preset }}
 
+    - name: Generate ya.make with affected test suites and modules list
+      if: inputs.increment == 'true'
+      uses: ./.github/actions/graph_compare
+
+    - name: Change target in case of incremental (tests)
+      id: test_run_choice
+      shell: bash
+      run: |
+        if [ "${{ inputs.increment }}" == "true" ]; then
+          echo "target=." >> $GITHUB_OUTPUT
+        else
+          echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
+        fi
+
     - name: Build
       uses: ./.github/actions/build_ya
       id: build
@@ -85,25 +112,11 @@ runs:
         build_target: ${{ inputs.build_target }}
         build_preset: ${{ inputs.build_preset }}
         bazel_remote_uri: ${{  fromJSON( inputs.vars ).REMOTE_CACHE_URL || '' }}
-        bazel_remote_username: ${{ inputs.put_build_results_to_cache &&  fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
-        bazel_remote_password: ${{ inputs.put_build_results_to_cache &&  fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
+        bazel_remote_username: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
+        bazel_remote_password: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
+        put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
         link_threads: ${{ inputs.link_threads }}
-
-    - name: Generate ya.make with affected test suites list
-      if: inputs.run_tests == 'true' && inputs.increment == 'true'
-      uses: ./.github/actions/graph_compare
-
-    - name: Check if there's a list of tests to run
-      id: test_run_choice
-      shell: bash
-      run: |
-        if [ -f ya.make ];then
-          echo "target='.'" >> $GITHUB_OUTPUT
-          echo "Listed test targets: "
-          cat ya.make
-        else
-          echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
-        fi
+        additional_ya_make_args: ${{ inputs.additional_ya_make_args }}
 
     - name: Run tests
       uses: ./.github/actions/test_ya
@@ -117,6 +130,7 @@ runs:
         testman_url: ${{  fromJSON( inputs.vars ).TESTMO_URL }}
         testman_project_id: ${{  fromJSON( inputs.vars ).TESTMO_PROJECT_ID }}
         link_threads: ${{ inputs.link_threads }}
+        additional_ya_make_args: ${{ inputs.additional_ya_make_args }}
         test_threads: ${{ inputs.test_threads }}
 
     - name: Notify about failed build

.github/actions/build_ya/action.yml

@@ -8,6 +8,9 @@ inputs:
     required: true
     default: "relwithdebinfo"
     description: "debug, relwithdebinfo, release-asan, release-tsan, release, release-cmake14"
+  put_build_results_to_cache:
+    required: false
+    default: "true"
   bazel_remote_uri:
     required: false
     description: "bazel-remote endpoint"
@@ -21,6 +24,9 @@ inputs:
     required: false
     default: "8"
     description: "link threads count"
+  additional_ya_make_args:
+    type: string
+    default: ""
 outputs:
   success:
     value: ${{ steps.build.outputs.status }}
@@ -65,7 +71,7 @@ runs:
           extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
         fi
 
-        if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then
+        if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then
           extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
           extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}")
           extra_params+=(--bazel-remote-put --dist-cache-evict-bins --add-result .o --add-result .a)
@@ -107,6 +113,10 @@ runs:
             ;;
         esac
 
+        if [ ! -z "${{ inputs.additional_ya_make_args }}" ]; then
+          extra_params+=(${{ inputs.additional_ya_make_args }})
+        fi
+
         echo "::debug::get version"
         ./ya --version
 
@@ -140,8 +150,14 @@ runs:
       run: |
         set -x
         if [ "${{ steps.build.outputs.status }}" == "failed" ]; then
+          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+            -d '{"state":"failure","description":"The check has been failed","context":"build_${{inputs.build_preset}}"}'
           echo "Build failed. see the [build logs]($LOG_URL)." | .github/scripts/tests/comment-pr.py --fail
         else
+          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+            -d '{"state":"success","description":"The check has been completed successfully","context":"build_${{inputs.build_preset}}"}'
           echo "Build successful." | .github/scripts/tests/comment-pr.py --ok
         fi
 

.github/actions/test_ya/action.yml

@@ -24,6 +24,9 @@ inputs:
     required: false
     default: "12"
     description: "link threads count"
+  additional_ya_make_args:
+    type: string
+    default: ""
   testman_token:
     required: false
     description: "test manager auth token"
@@ -122,7 +125,7 @@ runs:
             EXTRA_TAG=""
             ;;
         esac
-        
+
         testmo automation:resources:add-link --name build --url "$RUN_URL" --resources testmo.json
         testmo automation:resources:add-field --name git-sha --type string --value "${GITHUB_SHA:0:7}" --resources testmo.json
         RUN_ID=$(
@@ -191,13 +194,20 @@ runs:
             ;;
         esac
 
+        if [ ! -z "${{ inputs.additional_ya_make_args }}" ]; then
+          params+=(${{ inputs.additional_ya_make_args }})
+        fi
+
+        # Also build targets which are not in tests' dependencies
+        params+=(--no-strip-idle-build-results)
+
         echo "::debug::get version"
         ./ya --version
 
         echo "Tests are running..." | .github/scripts/tests/comment-pr.py
 
         ./ya test ${{ inputs.build_target }} "${params[@]}" \
-          --stat --log-file "$LOG_DIR/ya_log.txt" --evlog-file "$LOG_DIR/ya_evlog.jsonl" -DCONSISTENT_DEBUG \
+          --stat --log-file "$LOG_DIR/ya_log2.txt" --evlog-file "$LOG_DIR/ya_evlog2.jsonl" -DCONSISTENT_DEBUG \
           --no-dir-outputs \
           --junit "$JUNIT_REPORT_XML" --output "$OUT_DIR" || (
             RC=$?
@@ -279,14 +289,25 @@ runs:
         mkdir $ARTIFACTS_DIR/summary/
 
         cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
-        
+
         .github/scripts/tests/generate-summary.py \
           --summary-out-path $ARTIFACTS_DIR/summary/ \
           --summary-url-prefix $S3_URL_PREFIX/summary/ \
           --test-history-url $TEST_HISTORY_URL \
           --build-preset "$BUILD_PRESET" \
+          --status-report-file statusrep.txt \
           "Tests" ya-test.html "$JUNIT_REPORT_XML"
 
+        teststatus=$(cat statusrep.txt)
+        if [[ $teststatus == "success" ]];then
+          testmessage="The check has been completed successfully"
+        else
+          testmessage="The check has been failed"
+        fi
+        curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+            -d '{"state":"'$teststatus'","description":"'"$testmessage"'","context":"test_${{inputs.build_preset}}"}'
+
     - name: sync test results to s3
       if: always()
       shell: bash

.github/config/muted_ya.txt

@@ -21,7 +21,9 @@ ydb/core/kqp/ut/scheme KqpOlap.OlapRead_GenericQuerys
 ydb/core/kqp/ut/scheme KqpOlap.OlapRead_StreamGenericQuery
 ydb/core/kqp/ut/scheme KqpOlap.OlapRead_UsesGenericQueryOnJoinWithDataShardTable
 ydb/core/kqp/ut/scheme KqpQueryService.TableSink_ReplaceFromSelectLargeOlap
+ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication
 ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter
+ydb/core/kqp/ut/scheme [15/50]*
 ydb/core/kqp/ut/scheme [44/50]*
 ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
 ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
@@ -42,21 +44,18 @@ ydb/library/actors/http/ut HttpProxy.TooLongHeader
 ydb/library/actors/http/ut sole*
 ydb/library/yql/providers/generic/connector/tests/datasource/ydb* *
 ydb/public/lib/ydb_cli/topic/ut TTopicReaderTests.TestRun_ReadOneMessage
-ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut RetryPolicy.RetryWithBatching
 ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut RetryPolicy.TWriteSession_TestBrokenPolicy
 ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.ConflictingWrites
 ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.WriteRead
 ydb/public/sdk/cpp/client/ydb_topic/ut TSettingsValidation.TestDifferentDedupParams
 ydb/public/sdk/cpp/client/ydb_topic/ut [0/10]*
 ydb/public/sdk/cpp/client/ydb_topic/ut [6/10]*
-ydb/public/sdk/cpp/client/ydb_topic/ut TxUsage::WriteToTopic_Demo_*
 ydb/services/datastreams/ut DataStreams.TestGetRecordsStreamWithSingleShard
 ydb/services/datastreams/ut DataStreams.TestPutRecordsWithRead
 ydb/services/datastreams/ut DataStreams.TestReservedConsumersMetering
 ydb/services/datastreams/ut DataStreams.TestReservedStorageMetering
 ydb/services/fq/ut_integration Yq_1.*
-ydb/services/keyvalue/ut KeyValueGRPCService.SimpleWriteRead
-ydb/services/keyvalue/ut KeyValueGRPCService.SimpleCopyUnexistedKey
+ydb/services/keyvalue/ut KeyValueGRPCService.*
 ydb/services/keyvalue/ut sole*
 ydb/services/persqueue_v1/ut TPersQueueTest.CheckACLForGrpcWrite
 ydb/services/persqueue_v1/ut TPersQueueTest.SetupLockSession
@@ -83,6 +82,8 @@ ydb/tests/functional/kqp/kqp_indexes ConsistentIndexRead.InteractiveTx
 ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach
 ydb/tests/functional/restarts test_restarts.py.*
 ydb/tests/functional/serverless test_serverless.py.test_database_with_disk_quotas[enable_alter_database_create_hive_first--*]
+ydb/tests/functional/sqs/multinode/test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[kick_tablets-*] 
+ydb/tests/functional/sqs/multinode/test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[stop_node*] 
 ydb/tests/functional/tenants test_dynamic_tenants.py.*
 ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
 ydb/tests/functional/tenants test_tenants.py.*

.github/scripts/graph_compare.sh

@@ -1,21 +1,21 @@
 
 # Compares build graphs for two given refs in the current directory git repo
-# Creates ya.make in the current directory listing affected ydb test suites
+# Creates ya.make in the current directory listing affected ydb targets
 # Parameters: base_commit_sha head_commit_sha
 
-set -e
+set -ex
 
 workdir=$(mktemp -d)
 echo Workdir: $workdir
 echo Checkout base commit...
 git checkout $1
 echo Build graph for base commit...
-./ya make -Gj0 -ttt ydb --build release -k --cache-tests | jq '.graph[] | select( ."node-type"=="test")' > $workdir/graph_base
+./ya make -Gj0 -ttt ydb --build release -k --cache-tests --no-strip-idle-build-results | jq '.graph[]' > $workdir/graph_base
 
 echo Checkout head commit...
 git checkout $2
 echo Build graph for head commit...
-./ya make -Gj0 -ttt ydb --build release -k --cache-tests | jq '.graph[] | select( ."node-type"=="test")' > $workdir/graph_head
+./ya make -Gj0 -ttt ydb --build release -k --cache-tests --no-strip-idle-build-results | jq '.graph[]' > $workdir/graph_head
 
 echo Generate lists of uids for base and head...
 cat $workdir/graph_base | jq '.uid' > $workdir/uid_base
@@ -24,17 +24,35 @@ cat $workdir/graph_head | jq '.uid' > $workdir/uid_head
 echo Create a list of changed uids in the head graph...
 (cat $workdir/uid_head;(cat $workdir/uid_base;cat $workdir/uid_head) | sort | uniq -u) | sort | uniq -d > $workdir/uids_new
 
+echo Create ya.make
+echo "" > ya.make
+
 echo Generate list of test shard names from the head graph based on the list of uids...
-cat $workdir/graph_head | jq -r --slurpfile uids $workdir/uids_new 'select( any( .uid; .==$uids[] )) | .kv.path' | sort | uniq > $workdir/testsuites
+cat $workdir/graph_head | jq -r --slurpfile uids $workdir/uids_new 'select( ."node-type"=="test") | select( any( .uid; .==$uids[] )) | .kv.path' | sort | uniq > $workdir/testsuites
 
 echo Number of test suites:
 cat $workdir/testsuites | wc -l
 
 echo Removing test suite name from the list to get target names...
 sed -E 's/\/[^/]*$//g;/^null$/d' $workdir/testsuites > $workdir/ts2
 
-echo Generating temp ya.make with recurses to all required tests...
-cat $workdir/ts2 | (echo 'RECURSE_FOR_TESTS(';cat;echo ')') > ya.make
+echo Append into ya.make RECURSE_FOR_TESTS to all required tests...
+cat $workdir/ts2 | (echo 'RECURSE_FOR_TESTS(';cat;echo ')') >> ya.make
+
+echo Generate list of module names from the head graph based on the list of uids...
+cat $workdir/graph_head | jq -r --slurpfile uids $workdir/uids_new 'select( ."target_properties"."module_type" != null) | select( any( .uid; .==$uids[] )) | .target_properties.module_dir' | sort | uniq > $workdir/modules
+
+echo Number of modules:
+cat $workdir/modules | wc -l
+
+echo Filter only modules in ydb
+cat $workdir/modules | { grep "^ydb" || true; } > $workdir/modules2
+
+echo Number of modules:
+cat $workdir/modules2 | wc -l
+
+echo Append into ya.make RECURSE to all required modules...
+cat $workdir/modules2 | (echo 'RECURSE(';cat;echo ')') >> ya.make
 
-# echo Running ya test...
-# ./ya make -A -R --build relwithdebinfo .
+echo "ya.make content:"
+cat ya.make