Merge remote-tracking branch 'origin/ip-udf-refactoring' into ip-udf-refactoring

# Conflicts:
#	ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h
#	ydb/library/yql/udfs/common/ip_base/lib/ya.make

Author: Vadim Averin

.github/CODEOWNERS

@@ -0,0 +1 @@
+/ydb/docs/ @ydb-platform/docs

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,18 @@
+### Changelog entry <!-- a user-readable short description of changes introduced in this PR -->
+
+...
+
+### Changelog category <!-- remove all except one -->
+
+* New feature
+* Experimental feature
+* Improvement
+* Performance improvement
+* Bugfix 
+* Backward incompatible change
+* Documentation (changelog entry is not required)
+* Not for changelog (changelog entry is not required)
+
+### Additional information
+
+...

.github/actions/build_ya/action.yml

@@ -28,13 +28,19 @@ runs:
     - name: Init
       id: init
       shell: bash
+      env:
+        build_preset: ${{ inputs.build_preset }}
       run: |
         echo "SHELLOPTS=xtrace" >> $GITHUB_ENV
         export TMP_DIR=$(pwd)/tmp_build
         echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
         rm -rf $TMP_DIR && mkdir $TMP_DIR
-    
+        
+        echo "BUILD_PRESET=$build_preset" >> $GITHUB_ENV
+        echo "GITHUB_TOKEN=${{ github.token }}" >> $GITHUB_ENV
+
     - name: build
+      id: build
       shell: bash
       run: |
         extra_params=()
@@ -85,22 +91,41 @@ runs:
         echo "::debug::get version"
         ./ya --version
         
+        echo "Build **{platform_name}-${BUILD_PRESET}** is running..." | .github/scripts/tests/comment-pr.py
+        
+        # to be sure
+        set -o pipefail
+        
         ./ya make -k --build "${build_type}" --force-build-depends -D'BUILD_LANGUAGES=CPP PY3 PY2 GO' -T --stat -DCONSISTENT_DEBUG \
           --log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \
           --cache-size 512G --link-threads "${{ inputs.link_threads }}" \
-          "${extra_params[@]}" || (
+          "${extra_params[@]}" |& tee $TMP_DIR/ya_make.log || (
               RC=$?
               echo "::debug::ya make RC=$RC"
+              echo "status=failed" >> $GITHUB_OUTPUT
           )
 
     - name: sync logs to s3
       if: always()
       shell: bash
       run: |
         echo "::group::s3-sync"
-        s3cmd sync --acl-private --no-progress --stats --no-check-md5 "$TMP_DIR/" "$S3_BUCKET_PATH/build_logs/"
+        s3cmd sync --acl-private --exclude="ya_make.log" --no-progress --stats --no-check-md5 "$TMP_DIR/" "$S3_BUCKET_PATH/build_logs/"
+        s3cmd sync --acl-public --no-progress --stats --no-check-md5 "$TMP_DIR/ya_make.log" "$S3_BUCKET_PATH/build_logs/"
         echo "::endgroup::"
 
+    - name: comment-build-status
+      if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+      shell: bash
+      run: |
+        log_url="$S3_URL_PREFIX/build_logs/ya_make.log"
+
+        if [ "${{ steps.build.outputs.status }}" == "failed" ]; then
+          echo "Build failed. see the [build logs]($log_url)." | .github/scripts/tests/comment-pr.py --fail
+        else
+          echo "Build successful." | .github/scripts/tests/comment-pr.py --ok
+        fi
+
     - name: show free space
       if: always()
       shell: bash

.github/actions/test_ya/action.yml

@@ -66,6 +66,8 @@ runs:
         echo "TESTMO_TOKEN=${{ inputs.testman_token }}" >> $GITHUB_ENV
         echo "TESTMO_URL=${{ inputs.testman_url }}" >> $GITHUB_ENV
         echo "SUMMARY_LINKS=$(mktemp)" >> $GITHUB_ENV
+        echo "GITHUB_TOKEN=${{ github.token }}" >> $GITHUB_ENV
+        echo "BUILD_PRESET=${{ inputs.build_preset }}" >> $GITHUB_ENV
     
     - name: prepare
       shell: bash
@@ -93,7 +95,6 @@ runs:
         BRANCH_TAG="$GITHUB_REF_NAME"
         ARCH="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
         
-        BUILD_PRESET="${{ inputs.build_preset }}"
         case "$BUILD_PRESET" in
           relwithdebinfo)
             TESTMO_SOURCE="ya-${ARCH}"
@@ -105,7 +106,7 @@ runs:
             TESTMO_SOURCE="ya-${ARCH}-${BUILD_PRESET/release-/}"
             ;;
           *)
-            echo "Invalid preset: ${{ inputs.build_preset }}"
+            echo "Invalid preset: $BUILD_PRESET"
             exit 1
             ;;
         esac
@@ -170,7 +171,7 @@ runs:
         )
         
         # FIXME: copy-paste from build_ya
-        case "${{ inputs.build_preset }}" in
+        case "$BUILD_PRESET" in
           debug)
             params+=(--build "debug")
             ;;
@@ -196,7 +197,7 @@ runs:
             )
             ;;
           *)
-            echo "Invalid preset: ${{ inputs.build_preset }}"
+            echo "Invalid preset: $BUILD_PRESET"
             exit 1
             ;;
         esac
@@ -213,6 +214,8 @@ runs:
         echo "::debug::get version"
         ./ya --version
         
+        echo "Tests are running..." | .github/scripts/tests/comment-pr.py
+        
         if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then
           echo "::debug::start tests"
 
@@ -300,20 +303,16 @@ runs:
     - name: write tests summary
       shell: bash
       if: always()
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
       run: |
         mkdir $ARTIFACTS_DIR/summary/
         
         cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
         
-        platform_name=$(uname | tr '[:upper:]' '[:lower:]')-$(arch)
-        
         .github/scripts/tests/generate-summary.py \
           --summary-out-path $ARTIFACTS_DIR/summary/ \
           --summary-url-prefix $S3_URL_PREFIX/summary/ \
           --test-history-url $TEST_HISTORY_URL \
-          --build-preset "${platform_name}-${{ inputs.build_preset }}" \
+          --build-preset "$BUILD_PRESET" \
           "Tests" ya-test.html "$JUNIT_REPORT_XML"
 
     - name: sync test results to s3

.github/config/muted_ya.txt

@@ -1,19 +1,51 @@
+ydb/core/blobstorage/dsproxy/ut TBlobStorageProxySequenceTest.TestBlock42PutWithChangingSlowDisk
 ydb/core/blobstorage/pdisk/ut TSectorMap.*
+ydb/core/blobstorage/ut_blobstorage CostMetricsGetBlock4Plus2.TestGet4Plus2BlockRequests10000Inflight1BlobSize1000
 ydb/core/blobstorage/ut_blobstorage Defragmentation.DoesItWork
 ydb/core/blobstorage/ut_blobstorage SpaceCheckForDiskReassign.*
 ydb/core/blobstorage/ut_blobstorage VDiskAssimilation.Test
 ydb/core/blobstorage/ut_blobstorage [6/10]*
+ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk ReadOnlyVDisk.TestStorageLoad
+ydb/core/cms/ut_sentinel TSentinelTests.BSControllerCantChangeStatus
+ydb/core/persqueue/ut [31/40]*
+ydb/core/persqueue/ut TPersQueueMirrorer.TestBasicRemote
+ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
 ydb/core/kqp/ut/federated_query/generic *
+ydb/core/kqp/ut/olap *
+ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
+ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
+ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
 ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.ForgetAfterFail
 ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.RebootForgetAfterFail
+ydb/core/tx/columnshard/engines/ut *
+ydb/core/tx/coordinator/ut Coordinator.RestoreTenantConfiguration
+ydb/core/tx/schemeshard/ut_split_merge TSchemeShardSplitBySizeTest.Merge1KShards
+ydb/library/actors/http/ut HttpProxy.TooLongHeader
+ydb/library/actors/http/ut sole*
 ydb/library/yql/sql/pg/ut PgSqlParsingAutoparam.AutoParamValues_DifferentTypes
-ydb/services/ydb/sdk_sessions_pool_ut YdbSdkSessionsPool.StressTestSync10
+ydb/library/yql/tests/sql/dq_file/part16 *
+ydb/library/yql/tests/sql/dq_file/part18 test.py.test[expr-cast_type_bind-default.txt-Analyze]
+ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.WriteRead
+ydb/services/persqueue_v1/ut TPersQueueTest.DirectRead*
+ydb/services/persqueue_v1/ut [3/10]*
+ydb/services/ydb/sdk_sessions_pool_ut YdbSdkSessionsPool.StressTestSync*
+ydb/services/ydb/table_split_ut YdbTableSplit.SplitByLoadWithReadsMultipleSplitsWithData
+ydb/services/ydb/ut YdbOlapStore.LogPagingAfter-NotNull
 ydb/tests/fq/s3 *
 ydb/tests/fq/yds test_metrics_cleanup.py.TestCleanup.test_cleanup[v1]
 ydb/tests/functional/audit *
+ydb/tests/functional/blobstorage test_replication.py.TestReplicationAfterNodesRestart.test_replication[mirror-3-dc]
 ydb/tests/functional/clickbench test.py.test_plans*
+ydb/tests/functional/clickbench test.py.test_run_determentistic[column]
 ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach
 ydb/tests/functional/postgresql test_postgres.py.TestPostgresSuite.test_postgres_suite*
+ydb/tests/functional/restarts test_restarts.py.*
+ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud*
+ydb/tests/functional/sqs/cloud test_yandex_cloud_queue_counters.py.TestYmqQueueCounters.test_purge_queue_counters
+ydb/tests/functional/sqs/common test_queue_counters.py.TestSqsGettingCounters.test_purge_queue_counters
 ydb/tests/functional/tenants test_dynamic_tenants.py.*
 ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
-ydb/tests/functional/tenants test_tenants.py.*
+ydb/tests/functional/tenants test_tenants.py.*
+ydb/tests/functional/ydb_cli test_ydb_scripting.py.TestScriptingServiceHelp.test_help
+ydb/tests/functional/ydb_cli test_ydb_scripting.py.TestScriptingServiceHelp.test_help_ex
+ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout

.github/docker/Dockerfile

@@ -4,52 +4,23 @@ FROM ubuntu:22.04 AS builder
 ARG DEBIAN_FRONTEND=noninteractive
 ENV TZ=Etc/UTC
 
-
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends git wget gnupg lsb-release curl xz-utils tzdata \
-    cmake python3-dev python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make \
-    clang-12 lld-12 llvm-12 clang-14 lld-14 llvm-14 file \
-    && pip3 install conan==1.59 grpcio-tools pyinstaller==5.13.2 six pyyaml packaging PyHamcrest cryptography \
-    && (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
-         tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
-
+    && apt-get install -y --no-install-recommends \
+      python3 curl lsb-release python3-dev libc6-dev libidn11-dev libaio1 libaio-dev libgcc-11-dev binutils tree git
 
 COPY ydb/ /ydbwork/ydb/
 
-WORKDIR /ydbwork/
-
-RUN --mount=type=secret,id=ccache_remote_storage \
-    mkdir build && cd build \
-    && export CONAN_USER_HOME=/ydbwork/build \
-    && export CCACHE_BASEDIR=/ydbwork/ \
-    && export CCACHE_SLOPPINESS=locale \
-    && export CCACHE_REMOTE_STORAGE="$(cat /run/secrets/ccache_remote_storage)" \
-    && export CC=/usr/bin/clang-14 \
-    && export CC_FOR_BUILD=$CC \
-    && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \
-        -DCCACHE_PATH=/usr/local/bin/ccache \
-        -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \
-        ../ydb \
-    && ninja ydb/apps/ydbd/ydbd ydb/apps/ydb/all \
-    && ccache -s \
-    && strip ydb/apps/ydbd/ydbd \
-    && strip ydb/apps/ydb/ydb \
-    && mv ./ydb/apps/ydbd/ydbd / \
-    && mv ./ydb/apps/ydb/ydb / \
-    && /ydbd -V  \
-    && /ydb version \
-    && cd .. && rm -rf build  # for reduce cache size
-
+WORKDIR /ydbwork/ydb/
 
-
-# always use local_ydb.spec from main revision
-COPY main/ydb/public/tools/local_ydb/local_ydb.spec ydb/ydb/public/tools/local_ydb/
-
-RUN cd ydb && \
-    ./ydb/tests/oss/launch/compile_protos.sh . ydb library/cpp/actors && \
-    cd ydb/public/tools/local_ydb/ && \
-    pyinstaller local_ydb.spec && \
-    ./dist/local_ydb --help
+RUN ./ya make -r -T \
+      -C ydb/apps/ydbd/ydbd \
+      -C ydb/apps/ydb/ydb \
+      -C ydb/public/tools/local_ydb/local_ydb \
+      -o /ydbwork/out/ \
+    && /ydbwork/out/ydb/apps/ydbd/ydbd -V \
+    && tree /ydbwork/out/ \
+    && strip /ydbwork/out/ydb/apps/ydbd/ydbd /ydbwork/out/ydb/apps/ydb/ydb /ydbwork/out/ydb/public/tools/local_ydb/local_ydb \
+    && rm -rf ~/.ya  || echo 'unable to delete ~/.ya' # for reduce cache size
 
 
 FROM ubuntu:22.04
@@ -63,9 +34,11 @@ RUN mkdir -p /root/ydb/bin/ \
     && echo '{"check_version":false}' > root/ydb/bin/config.json
 
 COPY main/.github/docker/files/ /
-COPY --from=builder /ydbwork/ydb/ydb/public/tools/local_ydb/dist/local_ydb /
-COPY --from=builder /ydbd /ydb /
-
+COPY --from=builder \
+    /ydbwork/out/ydb/apps/ydbd/ydbd \
+    /ydbwork/out/ydb/apps/ydb/ydb \
+    /ydbwork/out/ydb/public/tools/local_ydb/local_ydb \
+    /
 
 # YDB grpc
 EXPOSE ${GRPC_TLS_PORT:-2135}

.github/docker/files/initialize_local_ydb

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -x
 
 INIT_YDB_SCRIPT=/init_ydb
 

.github/scripts/tests/comment-pr.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+import os
+import json
+import argparse
+from github import Github, Auth as GithubAuth
+from github.PullRequest import PullRequest
+from gh_status import update_pr_comment_text
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--rewrite", dest="rewrite", action="store_true")
+    parser.add_argument("--color", dest="color", default="white")
+    parser.add_argument("--fail", dest="fail", action="store_true")
+    parser.add_argument("--ok", dest="ok", action="store_true")
+    parser.add_argument("text", type=argparse.FileType("r"), nargs="?", default="-")
+
+    args = parser.parse_args()
+    color = args.color
+
+    if args.ok:
+        color = 'green'
+    elif args.fail:
+        color = 'red'
+
+    build_preset = os.environ["BUILD_PRESET"]
+
+    gh = Github(auth=GithubAuth.Token(os.environ["GITHUB_TOKEN"]))
+
+    with open(os.environ["GITHUB_EVENT_PATH"]) as fp:
+        event = json.load(fp)
+
+    pr = gh.create_from_raw_data(PullRequest, event["pull_request"])
+
+    update_pr_comment_text(pr, build_preset, color, args.text.read().rstrip(), args.rewrite)
+
+
+if __name__ == "__main__":
+    main()