Skip to content

Commit 89ce2f9

Browse files
maximyurchukmaximyurchuk
authored
Merge pull request #9960 from uzhastik/24_3_merge_3_1
q-stable-ydb 24 3 merge 3 1
2 parents e21f1de + 7141531 commit 89ce2f9

38 files changed

+1067
-246
lines changed

‎.github/actions/s3cmd/action.yml‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,16 @@ inputs:
88
required: true
99
description: "s3 key secret"
1010
s3_bucket:
11-
required: true
11+
required: false
1212
description: "s3 bucket"
1313
s3_endpoint:
1414
required: true
1515
description: "s3 endpoint"
1616
folder_prefix:
17-
required: true
17+
required: false
1818
description: "folder prefix"
1919
build_preset:
20-
required: true
20+
required: false
2121
description: "build preset like relwithdebinfo"
2222
runs:
2323
using: "composite"
@@ -35,7 +35,14 @@ runs:
3535
host_base = storage.yandexcloud.net
3636
host_bucket = %(bucket)s.storage.yandexcloud.net
3737
EOF
38-
38+
env:
39+
s3_key_id: ${{ inputs.s3_key_id }}
40+
s3_secret_access_key: ${{ inputs.s3_key_secret }}
41+
42+
- name: export s3 path variables
43+
shell: bash
44+
if: inputs.build_preset
45+
run: |
3946
folder="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
4047
4148
BUILD_PRESET="${{ inputs.build_preset }}"
@@ -57,7 +64,4 @@ runs:
5764
echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
5865
echo "S3_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
5966
echo "S3_TEST_ARTIFACTS_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
60-
echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
61-
env:
62-
s3_key_id: ${{ inputs.s3_key_id }}
63-
s3_secret_access_key: ${{ inputs.s3_key_secret }}
67+
echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV

‎.github/workflows/nightly_build.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Nightly-Build # workflow used to upload built binaries to s3
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
runner_label:
6+
type: string
7+
default: "auto-provisioned"
8+
description: "runner label"
9+
commit_sha:
10+
type: string
11+
default: ""
12+
jobs:
13+
build_and_test:
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
build_preset: ["relwithdebinfo", "release-asan"]
18+
runs-on: [ self-hosted, auto-provisioned, "${{ format('build-preset-{0}', matrix.build_preset) }}" ]
19+
name: Build and test ${{ matrix.build_preset }}
20+
steps:
21+
- name: Checkout
22+
uses: actions/checkout@v4
23+
with:
24+
ref: ${{ inputs.commit_sha }}
25+
fetch-depth: 2
26+
- name: Setup ydb access
27+
uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
28+
with:
29+
ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
30+
- name: Build and test
31+
uses: ./.github/actions/build_and_test_ya
32+
with:
33+
build_preset: ${{ matrix.build_preset }}
34+
build_target: "ydb/apps/ydbd"
35+
increment: false
36+
run_tests: false
37+
put_build_results_to_cache: false
38+
secs: ${{ format('{{"TESTMO_TOKEN2":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}',
39+
secrets.TESTMO_TOKEN2, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }}
40+
vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}',
41+
vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }}
42+
- name: Setup s3cmd
43+
uses: ./.github/actions/s3cmd
44+
with:
45+
s3_bucket: "ydb-builds"
46+
s3_endpoint: ${{ vars.AWS_ENDPOINT }}
47+
s3_key_id: ${{ secrets.AWS_KEY_ID }}
48+
s3_key_secret: ${{ secrets.AWS_KEY_VALUE }}
49+
50+
- name: sync results to s3 and publish links
51+
shell: bash
52+
run: |
53+
set -x
54+
s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "ydb/apps/ydbd/ydbd" "s3://ydb-builds/${{ github.ref_name }}/${{ matrix.build_preset }}/ydbd" -d

‎ydb/core/grpc_services/grpc_request_check_actor.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,7 @@ class TGrpcRequestCheckActor
312312
SetTokenAndDie();
313313
break;
314314
case Ydb::StatusIds::TIMEOUT:
315+
case Ydb::StatusIds::CANCELLED:
315316
Counters_->IncDatabaseRateLimitedCounter();
316317
LOG_INFO(*TlsActivationContext, NKikimrServices::GRPC_SERVER, "Throughput limit exceeded");
317318
ReplyOverloadedAndDie(MakeIssue(NKikimrIssues::TIssuesIds::YDB_RESOURCE_USAGE_LIMITED, "Throughput limit exceeded"));
@@ -331,7 +332,8 @@ class TGrpcRequestCheckActor
331332
}
332333
};
333334

334-
req.mutable_operation_params()->mutable_operation_timeout()->set_nanos(200000000); // same as cloud-go serverless proxy
335+
req.mutable_operation_params()->mutable_operation_timeout()->set_seconds(10);
336+
req.mutable_operation_params()->mutable_cancel_after()->set_nanos(200000000); // same as cloud-go serverless proxy
335337

336338
NKikimr::NRpcService::RateLimiterAcquireUseSameMailbox(
337339
std::move(req),

‎ydb/core/grpc_services/local_rate_limiter.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ TActorId RateLimiterAcquireUseSameMailbox(
2323
onSuccess();
2424
break;
2525
case Ydb::StatusIds::TIMEOUT:
26+
case Ydb::StatusIds::CANCELLED:
2627
onTimeout();
2728
break;
2829
default:
@@ -32,7 +33,8 @@ TActorId RateLimiterAcquireUseSameMailbox(
3233
};
3334

3435
Ydb::RateLimiter::AcquireResourceRequest request;
35-
SetDuration(duration, *request.mutable_operation_params()->mutable_operation_timeout());
36+
SetDuration(duration * 10, *request.mutable_operation_params()->mutable_operation_timeout());
37+
SetDuration(duration, *request.mutable_operation_params()->mutable_cancel_after());
3638
request.set_coordination_node_path(fullPath.CoordinationNode);
3739
request.set_resource_path(fullPath.ResourcePath);
3840
request.set_required(required);
@@ -72,6 +74,7 @@ TActorId RateLimiterAcquireUseSameMailbox(
7274
onSuccess();
7375
break;
7476
case Ydb::StatusIds::TIMEOUT:
77+
case Ydb::StatusIds::CANCELLED:
7578
onTimeout();
7679
break;
7780
default:
@@ -82,7 +85,8 @@ TActorId RateLimiterAcquireUseSameMailbox(
8285

8386
const auto& rlPath = maybeRlPath.GetRef();
8487
Ydb::RateLimiter::AcquireResourceRequest request;
85-
SetDuration(duration, *request.mutable_operation_params()->mutable_operation_timeout());
88+
SetDuration(duration * 10, *request.mutable_operation_params()->mutable_operation_timeout());
89+
SetDuration(duration, *request.mutable_operation_params()->mutable_cancel_after());
8690
request.set_coordination_node_path(rlPath.CoordinationNode);
8791
request.set_resource_path(rlPath.ResourcePath);
8892
request.set_required(required);

‎ydb/core/grpc_services/rpc_create_table.cpp‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,28 @@ class TCreateTableRPC : public TRpcSchemeRequestActor<TCreateTableRPC, TEvCreate
181181
return;
182182
}
183183

184+
StatusIds::StatusCode code = StatusIds::SUCCESS;
185+
TString error;
186+
187+
bool hasSerial = false;
188+
for (const auto& column : req->columns()) {
189+
switch (column.default_value_case()) {
190+
case Ydb::Table::ColumnMeta::kFromSequence: {
191+
auto* seqDesc = modifyScheme->MutableCreateIndexedTable()->MutableSequenceDescription()->Add();
192+
if (!FillSequenceDescription(*seqDesc, column.from_sequence(), code, error)) {
193+
NYql::TIssues issues;
194+
issues.AddIssue(NYql::TIssue(error));
195+
return Reply(code, issues, ctx);
196+
}
197+
hasSerial = true;
198+
break;
199+
}
200+
default: break;
201+
}
202+
}
203+
184204
NKikimrSchemeOp::TTableDescription* tableDesc = nullptr;
185-
if (req->indexesSize()) {
205+
if (req->indexesSize() || hasSerial) {
186206
modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateIndexedTable);
187207
tableDesc = modifyScheme->MutableCreateIndexedTable()->MutableTableDescription();
188208
} else {
@@ -192,9 +212,6 @@ class TCreateTableRPC : public TRpcSchemeRequestActor<TCreateTableRPC, TEvCreate
192212

193213
tableDesc->SetName(name);
194214

195-
StatusIds::StatusCode code = StatusIds::SUCCESS;
196-
TString error;
197-
198215
if (!FillColumnDescription(*tableDesc, req->columns(), code, error)) {
199216
NYql::TIssues issues;
200217
issues.AddIssue(NYql::TIssue(error));

‎ydb/core/grpc_services/rpc_describe_table.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,14 @@ class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDe
142142
return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx);
143143
}
144144

145+
StatusIds::StatusCode code = StatusIds::SUCCESS;
146+
TString error;
147+
if (!FillSequenceDescription(describeTableResult, tableDescription, code, error)) {
148+
LOG_ERROR(ctx, NKikimrServices::GRPC_SERVER, "Unable to fill sequence description: %s", error.c_str());
149+
Request_->RaiseIssue(NYql::TIssue(error));
150+
return Reply(Ydb::StatusIds::INTERNAL_ERROR, ctx);
151+
}
152+
145153
describeTableResult.mutable_primary_key()->CopyFrom(tableDescription.GetKeyColumnNames());
146154

147155
try {
@@ -212,6 +220,10 @@ class TDescribeTableRPC : public TRpcSchemeRequestActor<TDescribeTableRPC, TEvDe
212220
record->MutableOptions()->SetReturnPartitionStats(true);
213221
}
214222

223+
if (req->include_set_val()) {
224+
record->MutableOptions()->SetReturnSetVal(true);
225+
}
226+
215227
record->MutableOptions()->SetShowPrivateTable(ShowPrivatePath(path));
216228

217229
ctx.Send(MakeTxProxyID(), navigateRequest.release());

‎ydb/core/grpc_services/rpc_rate_limiter_api.cpp‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -594,11 +594,18 @@ class TAcquireRateLimiterResourceRPC : public TRateLimiterRequest<TAcquireRateLi
594594
SendRequest();
595595
}
596596

597+
// Always race when "cancel after" time is not set.
598+
// If "cancel after" is not set, quoter service can spend resource and say "OK", but we here reply with TIMEOUT.
597599
void OnOperationTimeout(const TActorContext& ctx) {
598600
Send(MakeQuoterServiceID(), new TEvQuota::TEvRpcTimeout(GetProtoRequest()->coordination_node_path(), GetProtoRequest()->resource_path()), 0, 0);
599601
TBase::OnOperationTimeout(ctx);
600602
}
601603

604+
// Do nothing here, because quoter service replies after "cancel after" time passes.
605+
void OnCancelOperation(const TActorContext& ctx) {
606+
Y_UNUSED(ctx);
607+
}
608+
602609
STFUNC(StateFunc) {
603610
switch (ev->GetTypeRewrite()) {
604611
hFunc(TEvQuota::TEvClearance, Handle);
@@ -637,22 +644,37 @@ class TAcquireRateLimiterResourceRPC : public TRateLimiterRequest<TAcquireRateLi
637644
true));
638645
}
639646

647+
StatusIds::StatusCode QuoterDeadlineStatusCode() {
648+
if (const TDuration cancelAfter = GetCancelAfter(); cancelAfter && cancelAfter < GetOperationTimeout()) {
649+
return StatusIds::CANCELLED;
650+
}
651+
return StatusIds::TIMEOUT;
652+
}
653+
640654
void SendLeaf(const TEvQuota::TResourceLeaf& leaf) {
655+
TDuration deadline = GetOperationTimeout();
656+
// CancelAfter is an intelligent way to say quoter service that we can wait maximum time.
657+
// After that time quoter service sends EResult::Deadline.
658+
// It says that the system lacks the resource.
659+
if (const TDuration cancelAfter = GetCancelAfter(); cancelAfter && cancelAfter < deadline) {
660+
deadline = cancelAfter;
661+
}
662+
641663
Send(MakeQuoterServiceID(),
642-
new TEvQuota::TEvRequest(TEvQuota::EResourceOperator::And, { leaf }, GetOperationTimeout()), 0, 0);
664+
new TEvQuota::TEvRequest(TEvQuota::EResourceOperator::And, { leaf }, deadline), 0, 0);
643665
}
644666

645667
void Handle(TEvQuota::TEvClearance::TPtr& ev) {
646668
switch (ev->Get()->Result) {
647669
case TEvQuota::TEvClearance::EResult::Success:
648670
Reply(StatusIds::SUCCESS, TActivationContext::AsActorContext());
649-
break;
671+
break;
650672
case TEvQuota::TEvClearance::EResult::UnknownResource:
651673
Reply(StatusIds::BAD_REQUEST, TActivationContext::AsActorContext());
652-
break;
674+
break;
653675
case TEvQuota::TEvClearance::EResult::Deadline:
654-
Reply(StatusIds::TIMEOUT, TActivationContext::AsActorContext());
655-
break;
676+
Reply(QuoterDeadlineStatusCode(), TActivationContext::AsActorContext());
677+
break;
656678
default:
657679
Reply(StatusIds::INTERNAL_ERROR, TActivationContext::AsActorContext());
658680
}

‎ydb/core/http_proxy/http_req.cpp‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,7 @@ namespace NKikimr::NHttpProxy {
553553
.Counters = nullptr,
554554
.AWSSignature = std::move(HttpContext.GetSignature()),
555555
.IAMToken = HttpContext.IamToken,
556-
.FolderID = ""
556+
.FolderID = HttpContext.FolderId
557557
};
558558

559559
auto authRequestProxy = MakeHolder<NSQS::THttpProxyAuthRequestProxy>(
@@ -1148,10 +1148,15 @@ namespace NKikimr::NHttpProxy {
11481148
SourceAddress = address;
11491149
}
11501150

1151-
DatabasePath = Request->URL;
1151+
DatabasePath = Request->URL.Before('?');
11521152
if (DatabasePath == "/") {
11531153
DatabasePath = "";
11541154
}
1155+
auto params = TCgiParameters(Request->URL.After('?'));
1156+
if (auto it = params.Find("folderId"); it != params.end()) {
1157+
FolderId = it->second;
1158+
}
1159+
11551160
//TODO: find out databaseId
11561161
ParseHeaders(Request->Headers);
11571162
}

0 commit comments

Comments
 (0)