Skip to content

Commit 7efac58

Browse files
authored
Merge branch 'main' into longtx2
2 parents b25eb99 + b02657c commit 7efac58

File tree

18,344 files changed

+123211
-4989061
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

18,344 files changed

+123211
-4989061
lines changed

.github/CODEOWNERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/ydb/docs/ @ydb-platform/docs

.github/actions/test_ya/action.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -253,8 +253,7 @@ runs:
253253
shell: bash
254254
run: |
255255
.github/scripts/tests/transform-ya-junit.py -i \
256-
--mu .github/config/muted_test.txt \
257-
--mf .github/config/muted_functest.txt \
256+
-m .github/config/muted_ya.txt \
258257
--ya-out "$OUT_DIR" \
259258
--log-url-prefix "$S3_URL_PREFIX/logs/" \
260259
--log-out-dir "$ARTIFACTS_DIR/logs/" \

.github/config/muted_test.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ ydb-core-blobstorage-ut_blobstorage/VDiskAssimilation::Test
66
ydb-core-tx-columnshard-ut_schema/TColumnShardTestSchema::ForgetAfterFail
77
ydb-core-tx-columnshard-ut_schema/TColumnShardTestSchema::RebootForgetAfterFail
88
ydb-core-kqp-ut-federated_query-generic/*
9-
ydb-core-client-ut/TClientTest::FollowerOfflineBoot
109
ydb-library-yql-sql-pg-ut/PgSqlParsingAutoparam::AutoParamValues_DifferentTypes
1110
ydb-core-blobstorage-ut_blobstorage/[6/10]*
1211
ydb/core/blobstorage/ut_blobstorage/Defragmentation::DoesItWork

.github/config/muted_ya.txt

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
ydb/core/blobstorage/pdisk/ut TSectorMap.*
2+
ydb/core/blobstorage/ut_blobstorage CostMetricsGetBlock4Plus2.TestGet4Plus2BlockRequests10000Inflight1BlobSize1000
3+
ydb/core/blobstorage/ut_blobstorage Defragmentation.DoesItWork
4+
ydb/core/blobstorage/ut_blobstorage SpaceCheckForDiskReassign.*
5+
ydb/core/blobstorage/ut_blobstorage VDiskAssimilation.Test
6+
ydb/core/blobstorage/ut_blobstorage [6/10]*
7+
ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk ReadOnlyVDisk.TestStorageLoad
8+
ydb/core/cms/ut_sentinel TSentinelTests.BSControllerCantChangeStatus
9+
ydb/core/persqueue/ut TPersQueueMirrorer.TestBasicRemote
10+
ydb/core/kqp/ut/federated_query/generic *
11+
ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
12+
ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
13+
ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.ForgetAfterFail
14+
ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.RebootForgetAfterFail
15+
ydb/core/tx/coordinator/ut Coordinator.RestoreTenantConfiguration
16+
ydb/library/yql/providers/generic/connector/tests test.py.test_select_positive_postgresql*
17+
ydb/library/yql/sql/pg/ut PgSqlParsingAutoparam.AutoParamValues_DifferentTypes
18+
ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.WriteRead
19+
ydb/services/persqueue_v1/ut TPersQueueTest.DirectRead*
20+
ydb/services/ydb/sdk_sessions_pool_ut YdbSdkSessionsPool.StressTestSync10
21+
ydb/services/ydb/table_split_ut YdbTableSplit.SplitByLoadWithReadsMultipleSplitsWithData
22+
ydb/services/ydb/ut YdbOlapStore.LogPagingAfter-NotNull
23+
ydb/tests/fq/s3 *
24+
ydb/tests/fq/yds test_metrics_cleanup.py.TestCleanup.test_cleanup[v1]
25+
ydb/tests/functional/audit *
26+
ydb/tests/functional/clickbench test.py.test_plans*
27+
ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach
28+
ydb/tests/functional/postgresql test_postgres.py.TestPostgresSuite.test_postgres_suite*
29+
ydb/tests/functional/restarts test_restarts.py.*
30+
ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud*
31+
ydb/tests/functional/tenants test_dynamic_tenants.py.*
32+
ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
33+
ydb/tests/functional/tenants test_tenants.py.*
34+
ydb/tests/functional/ydb_cli test_ydb_scripting.py.TestScriptingServiceHelp.test_help
35+
ydb/tests/functional/ydb_cli test_ydb_scripting.py.TestScriptingServiceHelp.test_help_ex

.github/docker/Dockerfile

Lines changed: 17 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -4,52 +4,23 @@ FROM ubuntu:22.04 AS builder
44
ARG DEBIAN_FRONTEND=noninteractive
55
ENV TZ=Etc/UTC
66

7-
87
RUN apt-get update \
9-
&& apt-get install -y --no-install-recommends git wget gnupg lsb-release curl xz-utils tzdata \
10-
cmake python3-dev python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make \
11-
clang-12 lld-12 llvm-12 clang-14 lld-14 llvm-14 file \
12-
&& pip3 install conan==1.59 grpcio-tools pyinstaller==5.13.2 six pyyaml packaging PyHamcrest cryptography \
13-
&& (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
14-
tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
15-
8+
&& apt-get install -y --no-install-recommends \
9+
python3 curl lsb-release python3-dev libc6-dev libidn11-dev libaio1 libaio-dev libgcc-11-dev binutils tree git
1610

1711
COPY ydb/ /ydbwork/ydb/
1812

19-
WORKDIR /ydbwork/
20-
21-
RUN --mount=type=secret,id=ccache_remote_storage \
22-
mkdir build && cd build \
23-
&& export CONAN_USER_HOME=/ydbwork/build \
24-
&& export CCACHE_BASEDIR=/ydbwork/ \
25-
&& export CCACHE_SLOPPINESS=locale \
26-
&& export CCACHE_REMOTE_STORAGE="$(cat /run/secrets/ccache_remote_storage)" \
27-
&& export CC=/usr/bin/clang-14 \
28-
&& export CC_FOR_BUILD=$CC \
29-
&& cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \
30-
-DCCACHE_PATH=/usr/local/bin/ccache \
31-
-DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \
32-
../ydb \
33-
&& ninja ydb/apps/ydbd/ydbd ydb/apps/ydb/all \
34-
&& ccache -s \
35-
&& strip ydb/apps/ydbd/ydbd \
36-
&& strip ydb/apps/ydb/ydb \
37-
&& mv ./ydb/apps/ydbd/ydbd / \
38-
&& mv ./ydb/apps/ydb/ydb / \
39-
&& /ydbd -V \
40-
&& /ydb version \
41-
&& cd .. && rm -rf build # for reduce cache size
42-
13+
WORKDIR /ydbwork/ydb/
4314

44-
45-
# always use local_ydb.spec from main revision
46-
COPY main/ydb/public/tools/local_ydb/local_ydb.spec ydb/ydb/public/tools/local_ydb/
47-
48-
RUN cd ydb && \
49-
./ydb/tests/oss/launch/compile_protos.sh . ydb library/cpp/actors && \
50-
cd ydb/public/tools/local_ydb/ && \
51-
pyinstaller local_ydb.spec && \
52-
./dist/local_ydb --help
15+
RUN ./ya make -r -T \
16+
-C ydb/apps/ydbd/ydbd \
17+
-C ydb/apps/ydb/ydb \
18+
-C ydb/public/tools/local_ydb/local_ydb \
19+
-o /ydbwork/out/ \
20+
&& /ydbwork/out/ydb/apps/ydbd/ydbd -V \
21+
&& tree /ydbwork/out/ \
22+
&& strip /ydbwork/out/ydb/apps/ydbd/ydbd /ydbwork/out/ydb/apps/ydb/ydb /ydbwork/out/ydb/public/tools/local_ydb/local_ydb \
23+
&& rm -rf ~/.ya || echo 'unable to delete ~/.ya' # for reduce cache size
5324

5425

5526
FROM ubuntu:22.04
@@ -63,9 +34,11 @@ RUN mkdir -p /root/ydb/bin/ \
6334
&& echo '{"check_version":false}' > root/ydb/bin/config.json
6435

6536
COPY main/.github/docker/files/ /
66-
COPY --from=builder /ydbwork/ydb/ydb/public/tools/local_ydb/dist/local_ydb /
67-
COPY --from=builder /ydbd /ydb /
68-
37+
COPY --from=builder \
38+
/ydbwork/out/ydb/apps/ydbd/ydbd \
39+
/ydbwork/out/ydb/apps/ydb/ydb \
40+
/ydbwork/out/ydb/public/tools/local_ydb/local_ydb \
41+
/
6942

7043
# YDB grpc
7144
EXPOSE ${GRPC_TLS_PORT:-2135}

.github/docker/files/initialize_local_ydb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#!/bin/bash
2+
set -x
23

34
INIT_YDB_SCRIPT=/init_ydb
45

.github/scripts/tests/generate-summary.py

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,10 @@ def add_line(self, line: TestSummaryLine):
144144
self.is_failed |= line.is_failed
145145
self.lines.append(line)
146146

147+
@property
148+
def is_empty(self):
149+
return len(self.lines) == 0
150+
147151
def render_line(self, items):
148152
return f"| {' | '.join(items)} |"
149153

@@ -249,8 +253,12 @@ def write_summary(summary: TestSummary):
249253
else:
250254
fp = sys.stdout
251255

252-
for line in summary.render(add_footnote=True):
253-
fp.write(f"{line}\n")
256+
if summary.is_empty:
257+
fp.write(":red_circle: Test run completed, no test results found. Please check build logs.")
258+
else:
259+
for line in summary.render(add_footnote=True):
260+
fp.write(f"{line}\n")
261+
254262
fp.write("\n")
255263

256264
if summary_fn:
@@ -267,6 +275,9 @@ def gen_summary(summary_url_prefix, summary_out_folder, paths):
267275
test_result = TestResult.from_junit(case)
268276
summary_line.add(test_result)
269277

278+
if not summary_line.tests:
279+
continue
280+
270281
report_url = f"{summary_url_prefix}{html_fn}"
271282

272283
render_testlist_html(summary_line.tests, os.path.join(summary_out_folder, html_fn))
@@ -277,8 +288,12 @@ def gen_summary(summary_url_prefix, summary_out_folder, paths):
277288

278289

279290
def get_comment_text(pr: PullRequest, summary: TestSummary, build_preset: str, test_history_url: str):
280-
281-
if summary.is_failed:
291+
if summary.is_empty:
292+
return [
293+
f":red_circle: **{build_preset}**: Test run completed, no test results found for commit {pr.head.sha}. "
294+
f"Please check build logs."
295+
]
296+
elif summary.is_failed:
282297
result = f":red_circle: **{build_preset}**: some tests FAILED"
283298
else:
284299
result = f":green_circle: **{build_preset}**: all tests PASSED"

.github/scripts/tests/transform-ya-junit.py

Lines changed: 27 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -17,34 +17,34 @@ def log_print(*args, **kwargs):
1717
class YaMuteCheck:
1818
def __init__(self):
1919
self.regexps = set()
20+
self.regexps = []
2021

21-
def add_unittest(self, fn):
22+
def load(self, fn):
2223
with open(fn, "r") as fp:
2324
for line in fp:
2425
line = line.strip()
25-
path, rest = line.split("/", maxsplit=1)
26-
path = path.replace("-", "/")
27-
rest = rest.replace("::", ".")
28-
self.populate(f"{path}/{rest}")
29-
30-
def add_functest(self, fn):
31-
with open(fn, "r") as fp:
32-
for line in fp:
33-
line = line.strip()
34-
line = line.replace("::", ".")
35-
self.populate(line)
36-
37-
def populate(self, line):
38-
pattern = pattern_to_re(line)
39-
40-
try:
41-
self.regexps.add(re.compile(pattern))
42-
except re.error:
43-
log_print(f"Unable to compile regex {pattern!r}")
44-
45-
def __call__(self, suitename, testname):
46-
for r in self.regexps:
47-
if r.match(f"{suitename}/{testname}"):
26+
try:
27+
testsuite, testcase = line.split(" ", maxsplit=1)
28+
except ValueError:
29+
log_print(f"SKIP INVALID MUTE CONFIG LINE: {line!r}")
30+
continue
31+
self.populate(testsuite, testcase)
32+
33+
def populate(self, testsuite, testcase):
34+
check = []
35+
36+
for p in (pattern_to_re(testsuite), pattern_to_re(testcase)):
37+
try:
38+
check.append(re.compile(p))
39+
except re.error:
40+
log_print(f"Unable to compile regex {p!r}")
41+
return
42+
43+
self.regexps.append(tuple(check))
44+
45+
def __call__(self, suite_name, test_name):
46+
for ps, pt in self.regexps:
47+
if ps.match(suite_name) and pt.match(test_name):
4848
return True
4949
return False
5050

@@ -176,8 +176,7 @@ def main():
176176
parser.add_argument(
177177
"-i", action="store_true", dest="save_inplace", default=False, help="modify input file in-place"
178178
)
179-
parser.add_argument("--mu", help="unittest mute config")
180-
parser.add_argument("--mf", help="functional test mute config")
179+
parser.add_argument("-m", help="muted test list")
181180
parser.add_argument("--log-url-prefix", default="./", help="url prefix for logs")
182181
parser.add_argument("--log-out-dir", help="symlink logs to specific directory")
183182
parser.add_argument(
@@ -194,11 +193,8 @@ def main():
194193

195194
mute_check = YaMuteCheck()
196195

197-
if args.mu:
198-
mute_check.add_unittest(args.mu)
199-
200-
if args.mf:
201-
mute_check.add_functest(args.mf)
196+
if args.m:
197+
mute_check.load(args.m)
202198

203199
transform(
204200
args.in_file,

.github/workflows/allowed_dirs.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ on:
66

77
jobs:
88
build:
9+
concurrency:
10+
group: check-allowed-dirs-${{ github.ref }}
11+
cancel-in-progress: true
912
runs-on: ubuntu-latest
10-
1113
steps:
1214
- uses: actions/checkout@v3
1315

.github/workflows/build_and_test_provisioned.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ jobs:
5555
steps:
5656
- name: Checkout
5757
uses: actions/checkout@v3
58+
with:
59+
ref: cmakebuild
5860
- name: Build
5961
uses: ./.github/actions/build
6062
if: inputs.run_build

0 commit comments

Comments
 (0)