Skip to content

Commit 28d46aa

Browse files
pixccpixcc
authored
Merge 3b6f686 into ee4ad94 
2 parents ee4ad94 + 3b6f686 commit 28d46aa

File tree

15 files changed

+201
-105
lines changed

15 files changed

+201
-105
lines changed

‎ydb/core/blobstorage/nodewarden/distconf.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@ namespace NKikimr::NStorage {
1919
STLOG(PRI_DEBUG, BS_NODE, NWDC00, "Bootstrap");
2020

2121
auto ns = NNodeBroker::BuildNameserverTable(Cfg->NameserviceConfig);
22-
auto ev = std::make_unique<TEvInterconnect::TEvNodesInfo>();
22+
auto nodes = MakeIntrusive<TIntrusiveVector<TEvInterconnect::TNodeInfo>>();
2323
for (const auto& [nodeId, item] : ns->StaticNodeTable) {
24-
ev->Nodes.emplace_back(nodeId, item.Address, item.Host, item.ResolveHost, item.Port, item.Location);
24+
nodes->emplace_back(nodeId, item.Address, item.Host, item.ResolveHost, item.Port, item.Location);
2525
}
26+
auto ev = std::make_unique<TEvInterconnect::TEvNodesInfo>(nodes);
2627
Send(SelfId(), ev.release());
2728

2829
// and subscribe for the node list too

‎ydb/core/fq/libs/actors/nodes_manager.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -259,14 +259,13 @@ class TNodesManagerActor : public NActors::TActorBootstrapped<TNodesManagerActor
259259
void HandleResponse(NFq::TEvInternalService::TEvHealthCheckResponse::TPtr& ev) {
260260
try {
261261
const auto& status = ev->Get()->Status.GetStatus();
262-
THolder<TEvInterconnect::TEvNodesInfo> nameServiceUpdateReq(new TEvInterconnect::TEvNodesInfo());
263262
if (!ev->Get()->Status.IsSuccess()) {
264263
ythrow yexception() << status << '\n' << ev->Get()->Status.GetIssues().ToString();
265264
}
266265
const auto& res = ev->Get()->Result;
267266

268-
auto& nodesInfo = nameServiceUpdateReq->Nodes;
269-
nodesInfo.reserve(res.nodes().size());
267+
auto nodesInfo = MakeIntrusive<TIntrusiveVector<TEvInterconnect::TNodeInfo>>();
268+
nodesInfo->reserve(res.nodes().size());
270269

271270
Peers.clear();
272271
std::set<ui32> nodeIds; // may be not unique
@@ -281,7 +280,7 @@ class TNodesManagerActor : public NActors::TActorBootstrapped<TNodesManagerActor
281280
node.active_workers(), node.memory_limit(), node.memory_allocated(), node.data_center()});
282281

283282
if (node.interconnect_port()) {
284-
nodesInfo.emplace_back(TEvInterconnect::TNodeInfo{
283+
nodesInfo->emplace_back(TEvInterconnect::TNodeInfo{
285284
node.node_id(),
286285
node.node_address(),
287286
node.hostname(), // host
@@ -297,8 +296,9 @@ class TNodesManagerActor : public NActors::TActorBootstrapped<TNodesManagerActor
297296
ServiceCounters.Counters->GetCounter("PeerCount", false)->Set(Peers.size());
298297
ServiceCounters.Counters->GetCounter("NodesHealthCheckOk", true)->Inc();
299298

300-
LOG_T("Send NodeInfo with size: " << nodesInfo.size() << " to DynamicNameserver");
301-
if (!nodesInfo.empty()) {
299+
LOG_T("Send NodeInfo with size: " << nodesInfo->size() << " to DynamicNameserver");
300+
if (!nodesInfo->empty()) {
301+
THolder<TEvInterconnect::TEvNodesInfo> nameServiceUpdateReq(new TEvInterconnect::TEvNodesInfo(nodesInfo));
302302
Send(GetNameserviceActorId(), nameServiceUpdateReq.Release());
303303
}
304304
} catch (yexception &e) {

‎ydb/core/health_check/health_check_ut.cpp‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -337,11 +337,15 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
337337
}
338338

339339
void SetLongHostValue(TEvInterconnect::TEvNodesInfo::TPtr* ev) {
340+
auto nodes = MakeIntrusive<TIntrusiveVector<TEvInterconnect::TNodeInfo>>((*ev)->Get()->Nodes);
340341
TString host(1000000, 'a');
341-
auto& pbRecord = (*ev)->Get()->Nodes;
342-
for (auto itIssue = pbRecord.begin(); itIssue != pbRecord.end(); ++itIssue) {
343-
itIssue->Host = host;
342+
for (auto it = nodes->begin(); it != nodes->end(); ++it) {
343+
it->Host = host;
344344
}
345+
auto newEv = IEventHandle::Downcast<TEvInterconnect::TEvNodesInfo>(
346+
new IEventHandle((*ev)->Recipient, (*ev)->Sender, new TEvInterconnect::TEvNodesInfo(nodes))
347+
);
348+
ev->Swap(newEv);
345349
}
346350

347351
Ydb::Monitoring::SelfCheckResult RequestHc(size_t const groupNumber, size_t const vdiscPerGroupNumber, bool const isMergeRecords = false, bool const largeSizeVdisksIssues = false) {

‎ydb/core/mind/bscontroller/impl.h‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1412,9 +1412,9 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
14121412
TNodeId NodeId;
14131413
TNodeLocation Location;
14141414

1415-
THostRecord(TEvInterconnect::TNodeInfo&& nodeInfo)
1415+
THostRecord(const TEvInterconnect::TNodeInfo& nodeInfo)
14161416
: NodeId(nodeInfo.NodeId)
1417-
, Location(std::move(nodeInfo.Location))
1417+
, Location(nodeInfo.Location)
14181418
{}
14191419
};
14201420

@@ -1423,11 +1423,13 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
14231423
THashMap<TNodeId, THostId> NodeIdToHostId;
14241424

14251425
public:
1426+
THostRecordMapImpl() = default;
1427+
14261428
THostRecordMapImpl(TEvInterconnect::TEvNodesInfo *msg) {
1427-
for (TEvInterconnect::TNodeInfo& nodeInfo : msg->Nodes) {
1429+
for (const TEvInterconnect::TNodeInfo& nodeInfo : msg->Nodes) {
14281430
const THostId hostId(nodeInfo.Host, nodeInfo.Port);
14291431
NodeIdToHostId.emplace(nodeInfo.NodeId, hostId);
1430-
HostIdToRecord.emplace(hostId, std::move(nodeInfo));
1432+
HostIdToRecord.emplace(hostId, nodeInfo);
14311433
}
14321434
}
14331435

@@ -1805,8 +1807,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
18051807

18061808
// For test purposes, required for self heal actor
18071809
void CreateEmptyHostRecordsMap() {
1808-
TEvInterconnect::TEvNodesInfo nodes;
1809-
HostRecords = std::make_shared<THostRecordMapImpl>(&nodes);
1810+
HostRecords = std::make_shared<THostRecordMapImpl>();
18101811
}
18111812

18121813
ui64 NextConfigTxSeqNo = 1;

‎ydb/core/mind/dynamic_nameserver.cpp‎

Lines changed: 58 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -66,13 +66,18 @@ void TDynamicNodeResolverBase::Handle(TEvNodeBroker::TEvResolvedNode::TPtr &ev,
6666

6767
if (rec.GetStatus().GetCode() != NKikimrNodeBroker::TStatus::OK) {
6868
// Reset proxy if node expired.
69-
if (exists)
69+
if (exists) {
7070
ResetInterconnectProxyConfig(NodeId, ctx);
71+
ListNodesCache->Invalidate(); // node was erased
72+
}
7173
ReplyWithErrorAndDie(ctx);
7274
return;
7375
}
7476

7577
TDynamicConfig::TDynamicNodeInfo node(rec.GetNode());
78+
if (!exists || !oldNode.EqualExceptExpire(node)) {
79+
ListNodesCache->Invalidate();
80+
}
7681

7782
// If ID is re-used by another node then proxy has to be reset.
7883
if (exists && !oldNode.EqualExceptExpire(node))
@@ -202,31 +207,40 @@ void TDynamicNameserver::ResolveDynamicNode(ui32 nodeId,
202207
reply->NodeId = nodeId;
203208
ctx.Send(ev->Sender, reply);
204209
} else {
205-
ctx.RegisterWithSameMailbox(new TDynamicNodeResolver(SelfId(), nodeId, DynamicConfigs[domain], ev, deadline));
210+
ctx.RegisterWithSameMailbox(new TDynamicNodeResolver(SelfId(), nodeId, DynamicConfigs[domain],
211+
ListNodesCache, ev, deadline));
206212
}
207213
}
208214

209215
void TDynamicNameserver::SendNodesList(const TActorContext &ctx)
210-
{
216+
{
211217
auto now = ctx.Now();
212-
for (auto &sender : ListNodesQueue) {
213-
THolder<TEvInterconnect::TEvNodesInfo> reply(new TEvInterconnect::TEvNodesInfo);
218+
if (ListNodesCache->NeedUpdate(now)) {
219+
auto newNodes = MakeIntrusive<TIntrusiveVector<TEvInterconnect::TNodeInfo>>();
220+
auto newExpire = now;
221+
214222
for (const auto &pr : StaticConfig->StaticNodeTable) {
215-
reply->Nodes.emplace_back(pr.first,
216-
pr.second.Address, pr.second.Host, pr.second.ResolveHost,
217-
pr.second.Port, pr.second.Location, true);
223+
newNodes->emplace_back(pr.first,
224+
pr.second.Address, pr.second.Host, pr.second.ResolveHost,
225+
pr.second.Port, pr.second.Location, true);
218226
}
219227

220228
for (auto &config : DynamicConfigs) {
221229
for (auto &pr : config->DynamicNodes) {
222-
if (pr.second.Expire > now)
223-
reply->Nodes.emplace_back(pr.first, pr.second.Address,
224-
pr.second.Host, pr.second.ResolveHost,
225-
pr.second.Port, pr.second.Location, false);
230+
if (pr.second.Expire > now) {
231+
newNodes->emplace_back(pr.first, pr.second.Address,
232+
pr.second.Host, pr.second.ResolveHost,
233+
pr.second.Port, pr.second.Location, false);
234+
newExpire = std::min(newExpire, pr.second.Expire);
235+
}
226236
}
227237
}
228238

229-
ctx.Send(sender, reply.Release());
239+
ListNodesCache->Update(newNodes, newExpire);
240+
}
241+
242+
for (auto &sender : ListNodesQueue) {
243+
ctx.Send(sender, new TEvInterconnect::TEvNodesInfo(ListNodesCache->GetNodes()));
230244
}
231245
ListNodesQueue.clear();
232246
}
@@ -282,15 +296,18 @@ void TDynamicNameserver::UpdateState(const NKikimrNodeBroker::TNodesInfo &rec,
282296
config->ExpiredNodes.emplace(node.GetNodeId(), info);
283297
}
284298

299+
ListNodesCache->Invalidate();
285300
config->Epoch = rec.GetEpoch();
286301
ctx.Schedule(config->Epoch.End - ctx.Now(),
287302
new TEvPrivate::TEvUpdateEpoch(domain, config->Epoch.Id + 1));
288303
} else {
289304
// Note: this update may be optimized to only include new nodes
290305
for (auto &node : rec.GetNodes()) {
291306
auto nodeId = node.GetNodeId();
292-
if (!config->DynamicNodes.contains(nodeId))
307+
if (!config->DynamicNodes.contains(nodeId)) {
293308
config->DynamicNodes.emplace(nodeId, node);
309+
ListNodesCache->Invalidate();
310+
}
294311
}
295312
config->Epoch = rec.GetEpoch();
296313
}
@@ -377,8 +394,8 @@ void TDynamicNameserver::Handle(TEvInterconnect::TEvGetNode::TPtr &ev, const TAc
377394
ctx.Send(ev->Sender, reply.Release());
378395
} else {
379396
const TInstant deadline = ev->Get()->Deadline;
380-
ctx.RegisterWithSameMailbox(new TDynamicNodeSearcher(SelfId(), nodeId, DynamicConfigs[domain], ev.Release(),
381-
deadline));
397+
ctx.RegisterWithSameMailbox(new TDynamicNodeSearcher(SelfId(), nodeId, DynamicConfigs[domain],
398+
ListNodesCache, ev.Release(), deadline));
382399
}
383400
}
384401
}
@@ -437,6 +454,7 @@ void TDynamicNameserver::Handle(NConsole::TEvConsole::TEvConfigNotificationReque
437454
auto newStaticConfig = BuildNameserverTable(config.GetNameserviceConfig());
438455
if (StaticConfig->StaticNodeTable != newStaticConfig->StaticNodeTable) {
439456
StaticConfig = std::move(newStaticConfig);
457+
ListNodesCache->Invalidate();
440458
for (const auto& subscriber : StaticNodeChangeSubscribers) {
441459
TActivationContext::Send(new IEventHandle(SelfId(), subscriber, new TEvInterconnect::TEvListNodes));
442460
}
@@ -478,5 +496,29 @@ TIntrusivePtr<TTableNameserverSetup> BuildNameserverTable(const NKikimrConfig::T
478496
return table;
479497
}
480498

499+
TListNodesCache::TListNodesCache()
500+
: Nodes(nullptr)
501+
, Expire(TInstant::Zero())
502+
{}
503+
504+
505+
void TListNodesCache::Update(TIntrusiveVector<TEvInterconnect::TNodeInfo>::TConstPtr newNodes, TInstant newExpire) {
506+
Nodes = newNodes;
507+
Expire = newExpire;
508+
}
509+
510+
void TListNodesCache::Invalidate() {
511+
Nodes = nullptr;
512+
Expire = TInstant::Zero();
513+
}
514+
515+
bool TListNodesCache::NeedUpdate(TInstant now) const {
516+
return Nodes == nullptr || now > Expire;
517+
}
518+
519+
TIntrusiveVector<TEvInterconnect::TNodeInfo>::TConstPtr TListNodesCache::GetNodes() const {
520+
return Nodes;
521+
}
522+
481523
} // NNodeBroker
482524
} // NKikimr

‎ydb/core/mind/dynamic_nameserver_impl.h‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,19 @@ struct TDynamicConfig : public TThrRefBase {
6868

6969
using TDynamicConfigPtr = TIntrusivePtr<TDynamicConfig>;
7070

71+
class TListNodesCache : public TSimpleRefCount<TListNodesCache> {
72+
public:
73+
TListNodesCache();
74+
75+
void Update(TIntrusiveVector<TEvInterconnect::TNodeInfo>::TConstPtr newNodes, TInstant newExpire);
76+
void Invalidate();
77+
bool NeedUpdate(TInstant now) const;
78+
TIntrusiveVector<TEvInterconnect::TNodeInfo>::TConstPtr GetNodes() const;
79+
private:
80+
TIntrusiveVector<TEvInterconnect::TNodeInfo>::TConstPtr Nodes;
81+
TInstant Expire;
82+
};
83+
7184
class TDynamicNodeResolverBase : public TActorBootstrapped<TDynamicNodeResolverBase> {
7285
public:
7386
using TBase = TActorBootstrapped<TDynamicNodeResolverBase>;
@@ -77,10 +90,12 @@ class TDynamicNodeResolverBase : public TActorBootstrapped<TDynamicNodeResolverB
7790
}
7891

7992
TDynamicNodeResolverBase(TActorId owner, ui32 nodeId, TDynamicConfigPtr config,
93+
TIntrusivePtr<TListNodesCache> listNodesCache,
8094
TAutoPtr<IEventHandle> origRequest, TInstant deadline)
8195
: Owner(owner)
8296
, NodeId(nodeId)
8397
, Config(config)
98+
, ListNodesCache(listNodesCache)
8499
, OrigRequest(origRequest)
85100
, Deadline(deadline)
86101
{
@@ -117,6 +132,7 @@ class TDynamicNodeResolverBase : public TActorBootstrapped<TDynamicNodeResolverB
117132
TActorId Owner;
118133
ui32 NodeId;
119134
TDynamicConfigPtr Config;
135+
TIntrusivePtr<TListNodesCache> ListNodesCache;
120136
TAutoPtr<IEventHandle> OrigRequest;
121137
const TInstant Deadline;
122138

@@ -127,8 +143,9 @@ class TDynamicNodeResolverBase : public TActorBootstrapped<TDynamicNodeResolverB
127143
class TDynamicNodeResolver : public TDynamicNodeResolverBase {
128144
public:
129145
TDynamicNodeResolver(TActorId owner, ui32 nodeId, TDynamicConfigPtr config,
146+
TIntrusivePtr<TListNodesCache> listNodesCache,
130147
TAutoPtr<IEventHandle> origRequest, TInstant deadline)
131-
: TDynamicNodeResolverBase(owner, nodeId, config, origRequest, deadline)
148+
: TDynamicNodeResolverBase(owner, nodeId, config, listNodesCache, origRequest, deadline)
132149
{
133150
}
134151

@@ -139,8 +156,9 @@ class TDynamicNodeResolver : public TDynamicNodeResolverBase {
139156
class TDynamicNodeSearcher : public TDynamicNodeResolverBase {
140157
public:
141158
TDynamicNodeSearcher(TActorId owner, ui32 nodeId, TDynamicConfigPtr config,
159+
TIntrusivePtr<TListNodesCache> listNodesCache,
142160
TAutoPtr<IEventHandle> origRequest, TInstant deadline)
143-
: TDynamicNodeResolverBase(owner, nodeId, config, origRequest, deadline)
161+
: TDynamicNodeResolverBase(owner, nodeId, config, listNodesCache, origRequest, deadline)
144162
{
145163
}
146164

@@ -179,6 +197,7 @@ class TDynamicNameserver : public TActorBootstrapped<TDynamicNameserver> {
179197

180198
TDynamicNameserver(const TIntrusivePtr<TTableNameserverSetup> &setup, ui32 resolvePoolId)
181199
: StaticConfig(setup)
200+
, ListNodesCache(MakeIntrusive<TListNodesCache>())
182201
, ResolvePoolId(resolvePoolId)
183202
{
184203
Y_ABORT_UNLESS(StaticConfig->IsEntriesUnique());
@@ -254,6 +273,8 @@ class TDynamicNameserver : public TActorBootstrapped<TDynamicNameserver> {
254273
TIntrusivePtr<TTableNameserverSetup> StaticConfig;
255274
std::array<TDynamicConfigPtr, DOMAINS_COUNT> DynamicConfigs;
256275
TVector<TActorId> ListNodesQueue;
276+
TIntrusivePtr<TListNodesCache> ListNodesCache;
277+
257278
std::array<TActorId, DOMAINS_COUNT> NodeBrokerPipes;
258279
// When ListNodes requests are sent to NodeBroker tablets this
259280
// bitmap indicates domains which didn't answer yet.

0 commit comments

Comments
 (0)