save the WriteId + PartitionId #1037
Conversation
Alek5andr-Kotov
commented
Jan 16, 2024
Alek5andr-Kotov
commented
- для пары WriteId, PartitionId создаётся служебная партиция
- идентификатор партиции сохраняется между рестартами таблетки PQ
- служебная партиция восстанавливается на старте таблетки PQ
- таблетка PQ подписывается на WriteId в LongTxService
|
Note This is an automated comment that will be appended during run. 🔴 linux-x86_64-relwithdebinfo: some tests FAILED for commit 054477b.
🔴 linux-x86_64-release-asan: some tests FAILED for commit 054477b.
|
| @@ -1023,6 +1023,14 @@ Y_UNIT_TEST_F(Cancel_Tx, TPQTabletFixture) | |||
| WaitForPQWriteTxs(); | |||
| } | |||
|
|
|||
| //Y_UNIT_TEST_F(Write_In_Tx, TPQTabletFixture) | |||
There was a problem hiding this comment.
done
f92290e
ydb/core/protos/pqconfig.proto
Outdated
| }; | ||
|
|
||
| message TTabletTxInfo { | ||
| optional uint64 LastStep = 2; | ||
| optional uint64 LastTxId = 3; | ||
| } | ||
|
|
||
| message TTabletTxWrites { | ||
| repeated uint64 WriteIds = 1; |
There was a problem hiding this comment.
done
9181228
ydb/core/persqueue/pq_impl.cpp
Outdated
| @@ -1210,17 +1317,29 @@ void TPersQueue::Handle(TEvPQ::TEvTabletCacheCounters::TPtr& ev, const TActorCon | |||
| << "Counters. CacheSize " << CacheCounters.CacheSizeBytes << " CachedBlobs " << CacheCounters.CacheSizeBlobs); | |||
| } | |||
|
|
|||
| bool TPersQueue::AllPartitionsInited() const | |||
| { | |||
| return PartitionsInited == (Partitions.size() + ShadowPartitions.size()); | |||
There was a problem hiding this comment.
Nope. You should track only regular partitions here
There was a problem hiding this comment.
done
048869b
| @@ -830,6 +832,101 @@ void TPersQueue::ReadTxInfo(const NKikimrClient::TKeyValueResponse::TReadResult& | |||
| LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, "Tablet " << TabletID() << " LastStep " << LastStep << " LastTxId " << LastTxId); | |||
| } | |||
|
|
|||
| void TPersQueue::InitPlanStep(const NKikimrPQ::TTabletTxInfo& info) | |||
There was a problem hiding this comment.
use default parameters info = {} and next func will be not required any more
There was a problem hiding this comment.
done
102de71
ydb/core/persqueue/pq_impl.cpp
Outdated
| LOG_INFO_S(ctx, NKikimrServices::PERSQUEUE, "Tablet " << TabletID() << " has a tx writes info"); | ||
|
|
||
| NKikimrPQ::TTabletTxWrites info; | ||
| Y_ABORT_UNLESS(info.ParseFromString(read.GetValue())); |
There was a problem hiding this comment.
done
fdb95e3
ydb/core/persqueue/pq_impl.cpp
Outdated
| HandleReserveBytesRequest(responseCookie, actorId, req, ctx, pipeClient, sender); | ||
| } | ||
|
|
||
| void TPersQueue::HandleWriteRequestForShadowPartition(const ui64 responseCookie, |
There was a problem hiding this comment.
done
4250769
ydb/core/persqueue/pq_impl.cpp
Outdated
| } else if (req.CmdWriteSize()) { | ||
| HandleWriteRequestForShadowPartition(responseCookie, req, ctx); | ||
| } else { | ||
| Y_ABORT("CmdGetOwnership, CmdReserveBytes or CmdWrite expected"); |
There was a problem hiding this comment.
reply with error, not abort!
There was a problem hiding this comment.
ydb/core/persqueue/pq_impl.cpp
Outdated
| @@ -3031,6 +3309,7 @@ void TPersQueue::EndWriteTxs(const NKikimrClient::TResponse& resp, | |||
|
|
|||
| SendReplies(ctx); | |||
| CheckChangedTxStates(ctx); | |||
| ForwardGetOwnershipToShadowPartitions(ctx); | |||
There was a problem hiding this comment.
Change this func to CreateSupportivePartitions
There was a problem hiding this comment.
done
74c832f
ydb/core/persqueue/pq_impl.cpp
Outdated
| partition.GetOwnershipRequests.emplace_back(params.Cookie, params.Request, params.Sender); | ||
|
|
||
| if (txWrite.LongTxSubscriptionStatus == NKikimrLongTxService::TEvLockStatus::STATUS_UNSPECIFIED) { | ||
| SubscribeWriteId(writeId, ctx); |
There was a problem hiding this comment.
Subscribe to write id when got new writeid, not when in persisted
There was a problem hiding this comment.
done
74c832f
| { | ||
| auto& record = ev->Get()->Record; | ||
| ui64 writeId = record.GetLockId(); | ||
| if (TxWrites.contains(writeId)) { |
There was a problem hiding this comment.
If lockStatus == NO TX and Writeid is presented in memory state and no propose in memory state - delete it and all partitions
There was a problem hiding this comment.
Change deletion of writeID to :
- Move write id to inmemory map to_be_deleted_write_id -> shadow_partitions
- Send to all shadow partitions event ClearAllAndDie, wait for results
- when all results for partitions in writeId received, shedule deletion of all info for write id in tablet state.
There was a problem hiding this comment.
We agreed to do it in a separate task. There I described the above algorithm
LOGBROKER-8871
Alek5andr-Kotov
force-pushed
the
LOGBROKER-8825
branch
from
9181228 to
3716756
Compare
|
⚪
|
|
⚪
|
Alek5andr-Kotov
force-pushed
the
LOGBROKER-8825
branch
from
ba50373 to
4240ed5
Compare
|
⚪
|
|
⚪
|
|
⚪ |
|
⚪ |
|
⚪ |
|
⚪ |
|
⚪
|
|
⚪
|
|
⚪
|
|
⚪
|
|
⚪
|
|
⚪
|
