Add pr tests run to q-stable

.github/workflows/build_and_test_provisioned.yml

@@ -24,6 +24,12 @@ on:
       extra_compile_flags:
         required: false
         type: string
+      checkout_ref:
+        required: false
+        type: string
+      ninja_target:
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       runner_label:
@@ -47,6 +53,12 @@ on:
       extra_compile_flags:
         required: false
         type: string
+      checkout_ref:
+        required: false
+        type: string
+      ninja_target:
+        required: false
+        type: string
 
 jobs:
   main:
@@ -56,14 +68,15 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v3
       with:
-        ref: cmakebuild
+        ref: ${{ inputs.checkout_ref }}
     - name: Build
       uses: ./.github/actions/build
       if: inputs.run_build
       with:
         sanitizer: ${{ inputs.sanitizer }}
         ccache_remote_path: ${{ vars.REMOTE_CACHE_URL && format('http://{0}{1}', secrets.REMOTE_CACHE_AUTH, vars.REMOTE_CACHE_URL) || ''}}
         extra_compile_flags: ${{ inputs.extra_compile_flags }}
+        ninja_target: ${{ inputs.ninja_target }}
     - name: Run tests
       uses: ./.github/actions/test
       with:

.github/workflows/build_and_test_ya.yml

@@ -27,11 +27,11 @@ on:
         description: "run tests"
       test_threads:
         type: string
-        default: 28
+        default: 56
         description: "Test threads count"
       link_threads:
         type: string
-        default: 8
+        default: 12
         description: "link threads count"
       test_size:
         type: string
@@ -65,60 +65,18 @@ jobs:
       with:
         ref: ${{ inputs.commit_sha }}
 
-    - name: comment-build-start
-      if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
-      shell: bash
-      env:
-        BUILD_PRESET: ${{ inputs.build_preset }}
-        GITHUB_TOKEN: ${{ github.token }}
-      run: |
-        jobs_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs"
-        # tricky: we are searching job with name that contains build_preset
-        check_url=$(curl -s $jobs_url | jq --arg n "$BUILD_PRESET" -r '.jobs[] | select(.name | contains($n)) | .html_url')
-
-        echo "Pre-commit [check]($check_url) for ${{ inputs.commit_sha }} has started." | .github/scripts/tests/comment-pr.py --rewrite
-
-    - name: Prepare s3cmd
-      uses: ./.github/actions/s3cmd
+    - name: Build and test
+      uses: ./.github/actions/build_and_test_ya
       with:
-        s3_bucket: ${{ vars.AWS_BUCKET }}
-        s3_endpoint: ${{ vars.AWS_ENDPOINT }}
-        s3_key_id: ${{ secrets.AWS_KEY_ID }}
-        s3_key_secret: ${{ secrets.AWS_KEY_VALUE }}
-        folder_prefix: ya-
         build_preset: ${{ inputs.build_preset }}
-
-    - name: Build
-      uses: ./.github/actions/build_ya
-      if: inputs.run_build
-      with:
         build_target: ${{ inputs.build_target }}
-        build_preset: ${{ inputs.build_preset }}
-        bazel_remote_uri: ${{ vars.REMOTE_CACHE_URL_YA || '' }}
-        bazel_remote_username: ${{ inputs.put_build_results_to_cache && secrets.REMOTE_CACHE_USERNAME || '' }}
-        bazel_remote_password: ${{ inputs.put_build_results_to_cache && secrets.REMOTE_CACHE_PASSWORD || '' }}
-        link_threads: ${{ inputs.link_threads }}
-
-    - name: Run tests
-      uses: ./.github/actions/test_ya
-      if: inputs.run_tests
-      with:
-        build_target: ${{ inputs.build_target }}
-        build_preset: ${{ inputs.build_preset }}
+        increment: false
+        run_tests: ${{ inputs.run_tests }}
         test_size: ${{ inputs.test_size }}
         test_type: ${{ inputs.test_type }}
-        testman_token: ${{ secrets.TESTMO_TOKEN }}
-        testman_url: ${{ vars.TESTMO_URL }}
-        testman_project_id: ${{ vars.TESTMO_PROJECT_ID }}
-        bazel_remote_uri: ${{ vars.REMOTE_CACHE_URL_YA || '' }}
-        bazel_remote_username: ${{ inputs.put_build_results_to_cache && secrets.REMOTE_CACHE_USERNAME || '' }}
-        bazel_remote_password: ${{ inputs.put_build_results_to_cache && secrets.REMOTE_CACHE_PASSWORD || '' }}
-        link_threads: ${{ inputs.link_threads }}
         test_threads: ${{ inputs.test_threads }}
-
-    - name: comment-if-cancel
-      if: cancelled() && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target')
-      env:
-        BUILD_PRESET: ${{ inputs.build_preset }}
-        GITHUB_TOKEN: ${{ github.token }}
-      run:  echo "Check cancelled" | .github/scripts/tests/comment-pr.py --color black
+        put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
+        secs: ${{ format('{{"TESTMO_TOKEN":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}', 
+          secrets.TESTMO_TOKEN, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }}
+        vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}',
+          vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }}

.github/workflows/build_and_test_ya_provisioned.yml

@@ -13,10 +13,12 @@ on:
         description: "Build preset"
         options:
           - debug
+          - release
           - relwithdebinfo
           - release-asan
           - release-tsan
           - release-msan
+          - release-cmake14
       test_size:
         type: choice
         default: "small,medium,large"
@@ -43,11 +45,11 @@ on:
         description: "run tests"
       test_threads:
         type: string
-        default: "28"
+        default: "56"
         description: "Test threads count"
       link_threads:
         type: string
-        default: "8"
+        default: "12"
         description: "link threads count"
       runner_label:
         type: string
@@ -78,11 +80,11 @@ on:
         default: true
       test_threads:
         type: string
-        default: 28
+        default: 56
         description: "Test threads count"
       link_threads:
         type: string
-        default: 8
+        default: 12
         description: "link threads count"
       runner_label:
         type: string
@@ -99,8 +101,7 @@ jobs:
     with:
       # FIXME: always use auto-provisioned here?
       runner_label: ${{ inputs.runner_label }}
-      # naive check for -asan, -tsan, -msan
-      runner_additional_label: ${{ contains(inputs.build_preset, '-') && format('build-preset-{0}', inputs.build_preset) || '' }}
+      runner_additional_label: ${{ format('build-preset-{0}', inputs.build_preset) }}
       build_target: ${{ inputs.build_target }}
       build_preset: ${{ inputs.build_preset }}
       run_build: ${{ inputs.run_build }}

.github/workflows/pr_check.yml

@@ -4,6 +4,8 @@ on:
     branches:
       - 'main'
       - 'stable-*'
+      - 'stream-nb-*'
+      - '*-stable-*'
     paths-ignore:
       - 'ydb/docs/**'
       - '*'
@@ -79,12 +81,21 @@ jobs:
         uses: actions/github-script@v6
         with:
           script: |
+            let externalContributorLabel = 'external';
+
             github.rest.issues.createComment({
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,
               body: 'Hi! Thank you for contributing!\nThe tests on this PR will run after a maintainer adds an `ok-to-test` label to this PR manually. Thank you for your patience!'
             });
+
+            github.rest.issues.addLabels({
+                ...context.repo,
+                issue_number: context.issue.number,
+                labels: [externalContributorLabel]
+            });
+
       - name: cleanup-test-label
         uses: actions/github-script@v6
         with:
@@ -186,16 +197,28 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        build_preset: ["relwithdebinfo", "release-asan"]
+        build_preset: ["relwithdebinfo", "release-asan", "release-clang14"]
+    runs-on: [ self-hosted, auto-provisioned, "${{ format('build-preset-{0}', matrix.build_preset) }}" ]
     name: Build and test ${{ matrix.build_preset }}
-    uses: ./.github/workflows/build_and_test_ya_provisioned.yml
-    with:
-      build_preset: ${{ matrix.build_preset }}
-      build_target: "ydb/"
-      test_size: "small,medium"
-      test_type: "unittest,py3test,py2test,pytest"
-      test_threads: 52
-      runner_label: auto-provisioned
-      put_build_results_to_cache: true
-      commit_sha: ${{ needs.check-running-allowed.outputs.commit_sha }}
-    secrets: inherit
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ needs.check-running-allowed.outputs.commit_sha }}
+        fetch-depth: 2
+    - name: Build and test
+      uses: ./.github/actions/build_and_test_ya
+      with:
+        build_preset: ${{ matrix.build_preset }}
+        build_target: "ydb/"
+        increment: true
+        run_tests: ${{ contains(fromJSON('["relwithdebinfo", "release-asan"]'), matrix.build_preset) }}
+        test_size: "small,medium"
+        test_type: "unittest,py3test,py2test,pytest"
+        test_threads: 52
+        put_build_results_to_cache: true
+        run_tests_if_build_fails: false
+        secs: ${{ format('{{"TESTMO_TOKEN":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}', 
+          secrets.TESTMO_TOKEN, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }}
+        vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}',
+          vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }}