Merge branch 'main' into inference_params

Author: evanevanevanevannnn

.github/actions/build_and_test_ya/action.yml

@@ -42,6 +42,9 @@ inputs:
   additional_ya_make_args:
     type: string
     default: ""
+  test_retry_count:
+    default: ""
+    description: "how many times to retry failed tests"
   secs:
     type: string
     default: ""
@@ -124,6 +127,7 @@ runs:
         bazel_remote_username: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
         bazel_remote_password: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
         put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
+        test_retry_count: ${{ inputs.test_retry_count }}
 
     - name: build_stats
       shell: bash

.github/actions/test_ya/action.yml

@@ -334,8 +334,6 @@ runs:
             IS_LAST_RETRY=1
           fi
 
-          s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$PUBLIC_DIR/" "$S3_BUCKET_PATH/"
-
           if [ $FAILED_TESTS_COUNT -gt 500 ]; then 
             IS_LAST_RETRY=1
             TOO_MANY_FAILED="Too many tests failed, NOT going to retry"
@@ -351,9 +349,17 @@ runs:
               --status_report_file statusrep.txt \
               --is_retry $IS_RETRY \
               --is_last_retry $IS_LAST_RETRY \
+              --comment_color_file summary_color.txt \
+              --comment_text_file summary_text.txt \
               "Tests" $CURRENT_PUBLIC_DIR/ya-test.html "$CURRENT_JUNIT_XML_PATH"
           fi
-            
+
+          s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$PUBLIC_DIR/" "$S3_BUCKET_PATH/"
+
+          if [ "${{ inputs.run_tests }}" = "true" ]; then
+            cat summary_text.txt | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py --color `cat summary_color.txt`
+          fi
+          
           # upload tests results to YDB
           ydb_upload_run_name="${TESTMO_RUN_NAME// /"_"}"
           result=`.github/scripts/upload_tests_results.py --test-results-file ${CURRENT_JUNIT_XML_PATH} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull $ydb_upload_run_name --job-name "${{ github.workflow }}" --job-id "${{ github.run_id }}" --branch ${GITHUB_REF_NAME}`

.github/config/muted_ya.txt

@@ -101,3 +101,4 @@ ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
 ydb/tests/functional/tenants test_tenants.py.*
 ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset*
 ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout
+ydb/core/kqp/ut/query KqpStats.SysViewClientLost

.github/scripts/tests/comment-pr.py

@@ -32,7 +32,13 @@ def main():
         event = json.load(fp)
 
     pr = gh.create_from_raw_data(PullRequest, event["pull_request"])
-    update_pr_comment_text(pr, build_preset, run_number, color, args.text.read().rstrip(), args.rewrite)
+    text = args.text.read()
+    if text.endswith("\n"):
+        # dirty hack because echo adds a new line 
+        # and 'echo | comment-pr.py' leads to an extra newline  
+        text = text[:-1]
+
+    update_pr_comment_text(pr, build_preset, run_number, color, text, args.rewrite)
 
 
 if __name__ == "__main__":

.github/scripts/tests/generate-summary.py

@@ -1,20 +1,14 @@
 #!/usr/bin/env python3
 import argparse
 import dataclasses
-import datetime
-import json
 import os
-import re
 import sys
 import traceback
-from github import Github, Auth as GithubAuth
-from github.PullRequest import PullRequest
 from enum import Enum
 from operator import attrgetter
-from typing import List, Optional, Dict
+from typing import List, Dict
 from jinja2 import Environment, FileSystemLoader, StrictUndefined
 from junit_utils import get_property_value, iter_xml_files
-from gh_status import update_pr_comment_text
 from get_test_history import get_test_history
 
 
@@ -328,7 +322,7 @@ def gen_summary(public_dir, public_dir_url, paths, is_retry: bool, build_preset)
     return summary
 
 
-def get_comment_text(pr: PullRequest, summary: TestSummary, summary_links: str, is_last_retry: bool)->tuple[str, list[str]]:
+def get_comment_text(summary: TestSummary, summary_links: str, is_last_retry: bool)->tuple[str, list[str]]:
     color = "red"
     if summary.is_failed:
         color = "red" if is_last_retry else "yellow"
@@ -379,6 +373,8 @@ def main():
     parser.add_argument('--status_report_file', required=False)
     parser.add_argument('--is_retry', required=True, type=int)
     parser.add_argument('--is_last_retry', required=True, type=int)
+    parser.add_argument('--comment_color_file', required=True)
+    parser.add_argument('--comment_text_file', required=True)
     parser.add_argument("args", nargs="+", metavar="TITLE html_out path")
     args = parser.parse_args()
 
@@ -397,21 +393,17 @@ def main():
     else:
         overall_status = "success"
 
-    if os.environ.get("GITHUB_EVENT_NAME") in ("pull_request", "pull_request_target"):
-        gh = Github(auth=GithubAuth.Token(os.environ["GITHUB_TOKEN"]))
-        run_number = int(os.environ.get("GITHUB_RUN_NUMBER"))
+    color, text = get_comment_text(summary, args.summary_links, is_last_retry=bool(args.is_last_retry))
 
-        with open(os.environ["GITHUB_EVENT_PATH"]) as fp:
-            event = json.load(fp)
+    with open(args.comment_color_file, "w") as f:
+        f.write(color)
 
-        pr = gh.create_from_raw_data(PullRequest, event["pull_request"])
-        color, text = get_comment_text(pr, summary, args.summary_links, is_last_retry=bool(args.is_last_retry))
+    with open(args.comment_text_file, "w") as f:
+        f.write('\n'.join(text))
+        f.write('\n')
 
-        update_pr_comment_text(pr, args.build_preset, run_number, color, text='\n'.join(text), rewrite=False)
-
-    if args.status_report_file:
-        with open(args.status_report_file, 'w') as fo:
-            fo.write(overall_status)
+    with open(args.status_report_file, "w") as f:
+        f.write(overall_status)
 
 
 if __name__ == "__main__":

.github/workflows/acceptance_run.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ]
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ inputs.commit_sha }}
 

.github/workflows/allowed_dirs.yml

@@ -11,7 +11,7 @@ jobs:
       cancel-in-progress: true
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Check dirs
       run: ${{github.workspace}}/.github/check_dirs.sh  ${{github.workspace}}

.github/workflows/build_analytics.yml

@@ -33,7 +33,7 @@ jobs:
     runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ]
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ inputs.commit_sha }}
 
@@ -49,7 +49,6 @@ jobs:
         build_target: ${{ inputs.build_target || 'ydb/apps/ydbd'}}
         increment: false
         run_tests: false
-        test_retry_count: 1
         put_build_results_to_cache: false
         additional_ya_make_args: "-DDUMP_LINKER_MAP -DCOMPILER_TIME_TRACE --add-result .json"
         secs: ${{ format('{{"TESTMO_TOKEN2":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}',

.github/workflows/build_and_test_provisioned.yml

@@ -48,7 +48,7 @@ jobs:
     runs-on: [ self-hosted, "${{ inputs.runner_label }}" ]
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ inputs.checkout_ref }}
     - name: Build

.github/workflows/build_and_test_ya.yml

@@ -65,7 +65,7 @@ jobs:
     runs-on: [ self-hosted, "${{ inputs.runner_label }}", "${{ inputs.runner_additional_label || inputs.runner_label }}"]
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ inputs.commit_sha }}