Merge remote-tracking branch 'origin/main' into generic-lookup-timeout-use-lib-retry

Author: yumkam

.github/TESTOWNERS

@@ -17,6 +17,7 @@
 
 #Column Tables Development Team @zverevgeny TEAM:@ydb-platform/cs
 /ydb/core/tx/columnshard @ydb-platform/cs
+/ydb/tests/olap @ydb-platform/cs
 
 #Distributed System Infrastructure Team @ijon TEAM:@ydb-platform/system-infra
 /ydb/core/mind/hive @ydb-platform/system-infra

.github/actions/rightlib_sync/action.yml

@@ -0,0 +1,35 @@
+name: rightlib sync
+description: Automatically sync rightlib branch into main
+inputs:
+  command:
+    required: true
+    description: "create-pr or check-pr"
+  repository:
+    required: true
+    description: "token for access GitHub"
+  gh_personal_access_token:
+    required: true
+    description: "token for access GitHub"
+runs:
+  using: "composite"
+  steps:
+    - name: install packages
+      shell: bash
+      run: |
+        pip install PyGithub==2.5.0
+
+    - name: configure
+      shell: bash
+      run: |
+        git config --global user.email "alex@ydb.tech"
+        git config --global user.name "Alexander Smirnov"
+
+    - name: run-command
+      shell: bash
+      env:
+        REPO: ${{ inputs.repository }}
+        TOKEN: ${{ inputs.gh_personal_access_token }}
+      run: |
+        cd ./ydb/ci/rightlib
+        ./sync-rightlib.py "${{ inputs.command }}"
+

.github/config/muted_ya.txt

@@ -8,7 +8,8 @@ export GRPC_TLS_PORT=${GRPC_TLS_PORT:-2135}
 export GRPC_PORT=${GRPC_PORT:-2136}
 export YDB_GRPC_TLS_DATA_PATH="/ydb_certs"
 
-/local_ydb deploy --ydb-working-dir /ydb_data --ydb-binary-path /ydbd --fixed-ports --dont-use-log-files;
+# Start local_ydb tool. Pass additional arguments for local_ydb
+/local_ydb deploy --ydb-working-dir /ydb_data --ydb-binary-path /ydbd --fixed-ports --dont-use-log-files "$@";
 
 if [ -f "$INIT_YDB_SCRIPT" ]; then
   sh "$INIT_YDB_SCRIPT";

.github/docker/files/initialize_local_ydb

@@ -3,13 +3,10 @@
 import configparser
 import datetime
 import os
-import posixpath
 import re
 import ydb
 import logging
 
-from get_diff_lines_of_file import get_diff_lines_of_file
-from mute_utils import pattern_to_re
 from transform_ya_junit import YaMuteCheck
 from update_mute_issues import (
     create_and_add_issue_to_project,
@@ -181,8 +178,9 @@ def apply_and_add_mutes(all_tests, output_path, mute_check):
             for test in all_tests
             if test.get('days_in_state') >= 1
             and test.get('flaky_today')
-            and (test.get('pass_count') + test.get('fail_count')) > 3
+            and (test.get('pass_count') + test.get('fail_count')) >= 3
             and test.get('fail_count') > 2
+            and test.get('fail_count')/(test.get('pass_count') + test.get('fail_count')) > 0.2 # <=80% success rate
         )
         flaky_tests = sorted(flaky_tests)
         add_lines_to_file(os.path.join(output_path, 'flaky.txt'), flaky_tests)
@@ -193,8 +191,9 @@ def apply_and_add_mutes(all_tests, output_path, mute_check):
             for test in all_tests
             if test.get('days_in_state') >= 1
             and test.get('flaky_today')
-            and (test.get('pass_count') + test.get('fail_count')) > 3
+            and (test.get('pass_count') + test.get('fail_count')) >= 3
             and test.get('fail_count') > 2
+            and test.get('fail_count')/(test.get('pass_count') + test.get('fail_count')) > 0.2 # <=80% success rate
         )
         ## тесты может запускаться 1 раз в день. если за последние 7 дней набирается трешход то мьютим
         ## падения сегодня более весомы ??  
@@ -296,7 +295,7 @@ def read_tests_from_file(file_path):
                 testsuite, testcase = line.split(" ", maxsplit=1)
                 result.append({'testsuite': testsuite, 'testcase': testcase, 'full_name': f"{testsuite}/{testcase}"})
             except ValueError:
-                log_print(f"cant parse line: {line!r}")
+                logging.warning(f"cant parse line: {line!r}")
                 continue
     return result
 
@@ -338,9 +337,8 @@ def create_mute_issues(all_tests, file_path):
     for item in prepared_tests_by_suite:
 
         title, body = generate_github_issue_title_and_body(prepared_tests_by_suite[item])
-        result = create_and_add_issue_to_project(
-            title, body, state='Muted', owner=prepared_tests_by_suite[item][0]['owner'].split('/', 1)[1]
-        )
+        owner_value = prepared_tests_by_suite[item][0]['owner'].split('/', 1)[1] if '/' in prepared_tests_by_suite[item][0]['owner'] else prepared_tests_by_suite[item][0]['owner']
+        result = create_and_add_issue_to_project(title, body, state='Muted', owner=owner_value)
         if not result:
             break
         else:
@@ -374,8 +372,6 @@ def mute_worker(args):
         credentials=ydb.credentials_from_env_variables(),
     ) as driver:
         driver.wait(timeout=10, fail_fast=True)
-        session = ydb.retry_operation_sync(lambda: driver.table_client.session().create())
-        tc_settings = ydb.TableClientSettings().with_native_date_in_result_sets(enabled=True)
 
         all_tests = execute_query(driver)
     if args.mode == 'update_muted_ya':

.github/scripts/tests/create_new_muted_ya.py

@@ -329,7 +329,6 @@ def generate_github_issue_title_and_body(test_data):
     test_run_history_link = f"{CURRENT_TEST_HISTORY_DASHBOARD}{test_run_history_params}"
 
     # owner
-    owner_link = f"[{owner}](https://github.com/orgs/ydb-platform/teams/{owner.split('/',1)[1]})"
     # Тело сообщения и кодирование
     body_template = (
         f"Mute:<!--mute_list_start-->\n"
@@ -449,7 +448,7 @@ def get_muted_tests_from_issues():
     issues = get_issues_and_tests_from_project(ORG_NAME, PROJECT_ID)
     muted_tests = {}
     for issue in issues:
-        if issues[issue]["status"] == "Muted":
+        if issues[issue]["status"] == "Muted" and issues[issue]["state"] != 'CLOSED':
             for test in issues[issue]['tests']:
                 if test not in muted_tests:
                     muted_tests[test] = []

.github/scripts/tests/update_mute_issues.py

@@ -0,0 +1,23 @@
+name: Automerge PR
+on:
+  schedule:
+    - cron: "30 1-23 * * *"   # Hourly during 1-23 interval for PR completion
+  workflow_dispatch:
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+jobs:
+  check-pr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github
+            ydb/ci/
+      - uses: ./.github/actions/rightlib_sync
+        with:
+          command: check-pr
+          repository: ${{ github.repository }}
+          gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

.github/workflows/automerge_pr.yaml

@@ -7,16 +7,47 @@ jobs:
   label-external-issues:
     name: Label issue from external user
     runs-on: ubuntu-latest
-    # https://docs.github.com/en/graphql/reference/enums#commentauthorassociation
-    if: ${{ !contains(fromJson('["MEMBER", "OWNER", "COLLABORATOR"]'), github.event.issue.author_association) }}
     steps:
       - name: add external label
         uses: actions/github-script@v7
         with:
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           script: |
-            github.rest.issues.addLabels({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              labels: ['external']
-            })
+            const issueAuthor = context.payload.issue.user.login
+            
+            if (context.repo.owner == issueAuthor) {
+              console.log("Issue author is here");
+              return
+            }
+            
+            const org = context.repo.owner;
+            
+            const isOrgMember = async function () {
+              try {
+                const response = await github.rest.orgs.checkMembershipForUser({
+                  org,
+                  username: issueAuthor,
+                });
+                return response.status == 204;
+              } catch (error) {
+                if (error.status && error.status == 404) {
+                  return false;
+                }
+                throw error;
+              }
+            }
+            
+            console.log(`Checking membership for user: ${issueAuthor} in organization: ${org}`);
+            
+            if (!await isOrgMember()) {
+              console.log(`User ${issueAuthor} is not a member of the organization.`)
+            
+              github.rest.issues.addLabels({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                labels: ['external']
+              })
+            } else {
+              console.log(`User ${issueAuthor} is a member of the organization.`)
+            }

.github/workflows/label_external_issues.yml

@@ -4,6 +4,7 @@ on:
     branches:
       - 'main'
       - 'stable-*'
+      - 'prestable-*'
       - 'stream-nb-*'
       - '*-stable-*'
       - 'dev-*'

.github/workflows/pr_check.yml

@@ -19,25 +19,25 @@ jobs:
           - build_preset: relwithdebinfo
             threads_count: 52
             timeout: 300
-            build_target: "ydb/tests/sql/ ydb/tests/sql/large"
+            build_target: "ydb/tests/sql/ ydb/tests/stress"
             test_size: small,medium,large
             test_type: unittest,py3test,py2test,pytest
           - build_preset: release-asan
             threads_count: 20
             timeout: 480
-            build_target: "ydb/tests/sql/ ydb/tests/sql/large"
+            build_target: "ydb/tests/sql/ ydb/tests/stress"
             test_size: small,medium,large
             test_type: unittest,py3test,py2test,pytest
           - build_preset: release-msan
             threads_count: 20
             timeout: 480
-            build_target: "ydb/tests/sql/ ydb/tests/sql/large"
+            build_target: "ydb/tests/sql/ ydb/tests/stress"
             test_size: small,medium,large
             test_type: unittest,py3test,py2test,pytest
           - build_preset: release-tsan
             threads_count: 10
             timeout: 600
-            build_target: "ydb/tests/sql/ ydb/tests/sql/large"
+            build_target: "ydb/tests/sql/ ydb/tests/stress"
             test_size: small,medium
             test_type: unittest,py3test,py2test,pytest