Skip to content

Commit aea6107

Browse files
BarkovBGBarkovBG
authored
[Blockstore] Stop compaction writes after StatusDiskSpaceLightOrange (#3594)
* handle StatusDiskSpaceLightOrange * resolve issues * fix build
1 parent 2ca9a5e commit aea6107

File tree

7 files changed

+84
-18
lines changed

7 files changed

+84
-18
lines changed

‎cloud/blockstore/libs/storage/partition/part_actor.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -667,12 +667,17 @@ void TPartitionActor::HandleCheckBlobstorageStatusResult(
667667
for (const auto& channel: Info()->Channels) {
668668
bool diskSpaceYellowMove = false;
669669
bool diskSpaceYellowStop = false;
670+
bool diskSpaceLightOrange = false;
670671
if (auto latestEntry = channel.LatestEntry()) {
671672
diskSpaceYellowMove = IsIn(msg->LightYellowMoveGroups, latestEntry->GroupID);
672673
diskSpaceYellowStop = IsIn(msg->YellowStopGroups, latestEntry->GroupID);
674+
diskSpaceLightOrange = IsIn(msg->LightOrangeGroups, latestEntry->GroupID);
673675
}
674676

675-
EChannelPermissions permissions = EChannelPermission::SystemWritesAllowed;
677+
EChannelPermissions permissions = {};
678+
if (!diskSpaceLightOrange) {
679+
permissions |= EChannelPermission::SystemWritesAllowed;
680+
}
676681
if (!diskSpaceYellowStop) {
677682
permissions |= EChannelPermission::UserWritesAllowed;
678683
}

‎cloud/blockstore/libs/storage/partition/part_actor_patchblob.cpp‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,10 +247,27 @@ STFUNC(TPatchBlobActor::StateWork)
247247

248248
EChannelPermissions StorageStatusFlags2ChannelPermissions(TStorageStatusFlags ssf)
249249
{
250+
/*
251+
YellowStop: Tablets switch to read-only mode. Only system writes are allowed.
252+
253+
LightOrange: Alert: "Tablets have not stopped". Compaction writes are not
254+
allowed if this flag is received.
255+
256+
PreOrange: VDisk switches to read-only mode.
257+
258+
Orange: All VDisks in the group switch to read-only mode.
259+
260+
Red: PDisk stops issuing chunks.
261+
262+
Black: Reserved for recovery.
263+
*/
264+
250265
const auto outOfSpaceMask = static_cast<NKikimrBlobStorage::EStatusFlags>(
251-
NKikimrBlobStorage::StatusDiskSpaceRed
266+
NKikimrBlobStorage::StatusDiskSpaceBlack
267+
| NKikimrBlobStorage::StatusDiskSpaceRed
252268
| NKikimrBlobStorage::StatusDiskSpaceOrange
253-
// no need to check StatusDiskSpaceBlack since BS won't accept any write requests in black state anyway
269+
| NKikimrBlobStorage::StatusDiskSpacePreOrange
270+
| NKikimrBlobStorage::StatusDiskSpaceLightOrange
254271
);
255272
if (ssf.Check(outOfSpaceMask)) {
256273
return {};

‎cloud/blockstore/libs/storage/partition/part_actor_writeblob.cpp‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -302,10 +302,27 @@ STFUNC(TWriteBlobActor::StateWork)
302302

303303
EChannelPermissions StorageStatusFlags2ChannelPermissions(TStorageStatusFlags ssf)
304304
{
305+
/*
306+
YellowStop: Tablets switch to read-only mode. Only system writes are allowed.
307+
308+
LightOrange: Alert: "Tablets have not stopped". Compaction writes are not
309+
allowed if this flag is received.
310+
311+
PreOrange: VDisk switches to read-only mode.
312+
313+
Orange: All VDisks in the group switch to read-only mode.
314+
315+
Red: PDisk stops issuing chunks.
316+
317+
Black: Reserved for recovery.
318+
*/
319+
305320
const auto outOfSpaceMask = static_cast<NKikimrBlobStorage::EStatusFlags>(
306-
NKikimrBlobStorage::StatusDiskSpaceRed
321+
NKikimrBlobStorage::StatusDiskSpaceBlack
322+
| NKikimrBlobStorage::StatusDiskSpaceRed
307323
| NKikimrBlobStorage::StatusDiskSpaceOrange
308-
// no need to check StatusDiskSpaceBlack since BS won't accept any write requests in black state anyway
324+
| NKikimrBlobStorage::StatusDiskSpacePreOrange
325+
| NKikimrBlobStorage::StatusDiskSpaceLightOrange
309326
);
310327
if (ssf.Check(outOfSpaceMask)) {
311328
return {};

‎cloud/blockstore/libs/storage/partition/part_ut.cpp‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4456,7 +4456,8 @@ Y_UNIT_TEST_SUITE(TPartitionTest)
44564456
}),
44574457
TVector<ui32>({
44584458
env.GetGroupIds()[1],
4459-
})
4459+
}),
4460+
TVector<ui32>()
44604461
);
44614462
partition.SendToPipe(std::move(request));
44624463
}
@@ -4480,7 +4481,8 @@ Y_UNIT_TEST_SUITE(TPartitionTest)
44804481
TVector<ui32>({
44814482
env.GetGroupIds()[0],
44824483
env.GetGroupIds()[1],
4483-
})
4484+
}),
4485+
TVector<ui32>()
44844486
);
44854487
partition.SendToPipe(std::move(request));
44864488
}
@@ -4497,6 +4499,7 @@ Y_UNIT_TEST_SUITE(TPartitionTest)
44974499
{
44984500
auto request =
44994501
std::make_unique<TEvTablet::TEvCheckBlobstorageStatusResult>(
4502+
TVector<ui32>(),
45004503
TVector<ui32>(),
45014504
TVector<ui32>()
45024505
);
@@ -5346,24 +5349,33 @@ Y_UNIT_TEST_SUITE(TPartitionTest)
53465349
TPartitionClient partition(*runtime);
53475350
partition.WaitReady();
53485351

5349-
runtime->SetObserverFunc(
5350-
StorageStateChanger(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove));
5352+
runtime->SetObserverFunc(StorageStateChanger(
5353+
NKikimrBlobStorage::StatusDiskSpaceLightYellowMove |
5354+
NKikimrBlobStorage::StatusDiskSpaceYellowStop));
53515355
partition.WriteBlocks(TBlockRange32::WithLength(0, 1024));
53525356

53535357
{
53545358
auto response = partition.Compaction();
53555359
UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetError().GetCode());
53565360
}
53575361

5362+
ui32 flags = NKikimrBlobStorage::StatusDiskSpaceLightYellowMove |
5363+
NKikimrBlobStorage::StatusDiskSpaceYellowStop;
5364+
53585365
const auto badFlags = {
5366+
NKikimrBlobStorage::StatusDiskSpaceLightOrange,
5367+
NKikimrBlobStorage::StatusDiskSpacePreOrange,
53595368
NKikimrBlobStorage::StatusDiskSpaceOrange,
53605369
NKikimrBlobStorage::StatusDiskSpaceRed,
5370+
NKikimrBlobStorage::StatusDiskSpaceBlack
53615371
};
53625372

53635373
for (const auto flag: badFlags) {
5374+
flags |= flag;
5375+
53645376
partition.RebootTablet();
53655377

5366-
runtime->SetObserverFunc(StorageStateChanger(flag));
5378+
runtime->SetObserverFunc(StorageStateChanger(flags));
53675379
partition.WriteBlocks(TBlockRange32::WithLength(0, 1024));
53685380

53695381
auto request = partition.CreateCompactionRequest();
@@ -5467,7 +5479,8 @@ Y_UNIT_TEST_SUITE(TPartitionTest)
54675479
auto request =
54685480
std::make_unique<TEvTablet::TEvCheckBlobstorageStatusResult>(
54695481
TVector<ui32>({env.GetGroupIds()[0], env.GetGroupIds()[1]}),
5470-
TVector<ui32>({env.GetGroupIds()[0], env.GetGroupIds()[1]})
5482+
TVector<ui32>({env.GetGroupIds()[0], env.GetGroupIds()[1]}),
5483+
TVector<ui32>()
54715484
);
54725485
partition.SendToPipe(std::move(request));
54735486
}
@@ -5530,7 +5543,8 @@ Y_UNIT_TEST_SUITE(TPartitionTest)
55305543
auto request =
55315544
std::make_unique<TEvTablet::TEvCheckBlobstorageStatusResult>(
55325545
TVector<ui32>({env.GetGroupIds()[0]}),
5533-
TVector<ui32>({env.GetGroupIds()[0]})
5546+
TVector<ui32>({env.GetGroupIds()[0]}),
5547+
TVector<ui32>()
55345548
);
55355549
partition.SendToPipe(std::move(request));
55365550
}

‎cloud/blockstore/libs/storage/partition2/part2_ut.cpp‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3106,7 +3106,8 @@ Y_UNIT_TEST_SUITE(TPartition2Test)
31063106
}),
31073107
TVector<ui32>({
31083108
env.GetGroupIds()[1],
3109-
})
3109+
}),
3110+
TVector<ui32>()
31103111
);
31113112
partition.SendToPipe(std::move(request));
31123113
}
@@ -3131,7 +3132,8 @@ Y_UNIT_TEST_SUITE(TPartition2Test)
31313132
TVector<ui32>({
31323133
env.GetGroupIds()[0],
31333134
env.GetGroupIds()[1],
3134-
})
3135+
}),
3136+
TVector<ui32>()
31353137
);
31363138
partition.SendToPipe(std::move(request));
31373139
}
@@ -3149,6 +3151,7 @@ Y_UNIT_TEST_SUITE(TPartition2Test)
31493151
{
31503152
auto request =
31513153
std::make_unique<TEvTablet::TEvCheckBlobstorageStatusResult>(
3154+
TVector<ui32>(),
31523155
TVector<ui32>(),
31533156
TVector<ui32>()
31543157
);
@@ -3298,7 +3301,8 @@ Y_UNIT_TEST_SUITE(TPartition2Test)
32983301
auto request =
32993302
std::make_unique<TEvTablet::TEvCheckBlobstorageStatusResult>(
33003303
TVector<ui32>({env.GetGroupIds()[0], env.GetGroupIds()[1]}),
3301-
TVector<ui32>({env.GetGroupIds()[0], env.GetGroupIds()[1]})
3304+
TVector<ui32>({env.GetGroupIds()[0], env.GetGroupIds()[1]}),
3305+
TVector<ui32>()
33023306
);
33033307
partition.SendToPipe(std::move(request));
33043308
}
@@ -3361,7 +3365,8 @@ Y_UNIT_TEST_SUITE(TPartition2Test)
33613365
auto request =
33623366
std::make_unique<TEvTablet::TEvCheckBlobstorageStatusResult>(
33633367
TVector<ui32>({env.GetGroupIds()[0]}),
3364-
TVector<ui32>({env.GetGroupIds()[0]})
3368+
TVector<ui32>({env.GetGroupIds()[0]}),
3369+
TVector<ui32>()
33653370
);
33663371
partition.SendToPipe(std::move(request));
33673372
}

‎contrib/ydb/core/base/tablet.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -760,10 +760,13 @@ struct TEvTablet {
760760
struct TEvCheckBlobstorageStatusResult : public TEventLocal<TEvCheckBlobstorageStatusResult, EvCheckBlobstorageStatusResult> {
761761
TVector<ui32> LightYellowMoveGroups;
762762
TVector<ui32> YellowStopGroups;
763+
TVector<ui32> LightOrangeGroups;
763764

764-
TEvCheckBlobstorageStatusResult(TVector<ui32> &&lightYellowMoveGroups, TVector<ui32> &&yellowStopGroups)
765+
TEvCheckBlobstorageStatusResult(TVector<ui32> &&lightYellowMoveGroups, TVector<ui32> &&yellowStopGroups,
766+
TVector<ui32> &&lightOrangeGroups)
765767
: LightYellowMoveGroups(std::move(lightYellowMoveGroups))
766768
, YellowStopGroups(std::move(yellowStopGroups))
769+
, LightOrangeGroups(std::move(lightOrangeGroups))
767770

768771
{}
769772
};

‎contrib/ydb/core/base/tablet_status_checker.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ class TTabletStatusCheckRequest : public TActorBootstrapped<TTabletStatusCheckRe
1414
ui32 RequestsLeft;
1515
TVector<ui32> LightYellowMoveGroups;
1616
TVector<ui32> YellowStopGroups;
17+
TVector<ui32> LightOrangeGroups;
1718

1819
void Handle(TEvBlobStorage::TEvStatusResult::TPtr &ev, const TActorContext &ctx) {
1920
const TEvBlobStorage::TEvStatusResult *msg = ev->Get();
@@ -25,10 +26,14 @@ class TTabletStatusCheckRequest : public TActorBootstrapped<TTabletStatusCheckRe
2526
if (msg->StatusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceYellowStop)) {
2627
YellowStopGroups.push_back(ev->Cookie);
2728
}
29+
if (msg->StatusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceLightOrange)) {
30+
LightOrangeGroups.push_back(ev->Cookie);
31+
}
2832

2933
if (RequestsLeft == 0) {
3034
ctx.Send(ReplyTo, new TEvTablet::TEvCheckBlobstorageStatusResult(std::move(LightYellowMoveGroups),
31-
std::move(YellowStopGroups)));
35+
std::move(YellowStopGroups),
36+
std::move(LightOrangeGroups)));
3237
return Die(ctx);
3338
}
3439
}

0 commit comments

Comments
 (0)