Skip to content

Commit 1cba451

Browse files
jepett0jepett0
authored
Restore the commit fixing a memory leak in AWS SDK (#8823)
1 parent 59ed83a commit 1cba451

File tree

26 files changed

+811
-316
lines changed

26 files changed

+811
-316
lines changed

‎ydb/core/driver_lib/run/kikimr_services_initializers.cpp

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,25 @@
244244

245245
#include <util/system/hostname.h>
246246

247+
#include <aws/core/Aws.h>
248+
249+
namespace {
250+
251+
struct TAwsApiGuard {
252+
TAwsApiGuard() {
253+
Aws::InitAPI(Options);
254+
}
255+
256+
~TAwsApiGuard() {
257+
Aws::ShutdownAPI(Options);
258+
}
259+
260+
private:
261+
Aws::SDKOptions Options;
262+
};
263+
264+
}
265+
247266
namespace NKikimr {
248267

249268
namespace NKikimrServicesInitializers {
@@ -2763,5 +2782,16 @@ void TGraphServiceInitializer::InitializeServices(NActors::TActorSystemSetup* se
27632782
TActorSetupCmd(NGraph::CreateGraphService(appData->TenantName), TMailboxType::HTSwap, appData->UserPoolId));
27642783
}
27652784

2785+
TAwsApiInitializer::TAwsApiInitializer(IGlobalObjectStorage& globalObjects)
2786+
: GlobalObjects(globalObjects)
2787+
{
2788+
}
2789+
2790+
void TAwsApiInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
2791+
Y_UNUSED(setup);
2792+
Y_UNUSED(appData);
2793+
GlobalObjects.AddGlobalObject(std::make_shared<TAwsApiGuard>());
2794+
}
2795+
27662796
} // namespace NKikimrServicesInitializers
27672797
} // namespace NKikimr

‎ydb/core/driver_lib/run/kikimr_services_initializers.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -625,5 +625,14 @@ class TGraphServiceInitializer : public IKikimrServicesInitializer {
625625
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
626626
};
627627

628+
class TAwsApiInitializer : public IServiceInitializer {
629+
IGlobalObjectStorage& GlobalObjects;
630+
631+
public:
632+
TAwsApiInitializer(IGlobalObjectStorage& globalObjects);
633+
634+
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
635+
};
636+
628637
} // namespace NKikimrServicesInitializers
629638
} // namespace NKikimr

‎ydb/core/driver_lib/run/run.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1676,6 +1676,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
16761676
sil->AddServiceInitializer(new TGraphServiceInitializer(runConfig));
16771677
}
16781678

1679+
if (serviceMask.EnableAwsService) {
1680+
sil->AddServiceInitializer(new TAwsApiInitializer(*this));
1681+
}
1682+
16791683
return sil;
16801684
}
16811685

‎ydb/core/driver_lib/run/service_mask.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ union TBasicKikimrServicesMask {
7979
bool EnableGraphService:1;
8080
bool EnableCompDiskLimiter:1;
8181
bool EnableGroupedMemoryLimiter:1;
82+
bool EnableAwsService:1;
8283
};
8384

8485
struct {

‎ydb/core/driver_lib/run/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ SRCS(
2121
)
2222

2323
PEERDIR(
24+
contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core
2425
contrib/libs/protobuf
2526
ydb/library/actors/core
2627
ydb/library/actors/dnsresolver

‎ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717

1818
#include <util/system/hostname.h>
1919
#include <library/cpp/deprecated/atomic/atomic.h>
20+
#include <library/cpp/testing/hook/hook.h>
21+
22+
#include <aws/core/Aws.h>
2023

2124
namespace NKikimr {
2225

@@ -32,6 +35,16 @@ enum class EInitialEviction {
3235

3336
namespace {
3437

38+
Aws::SDKOptions Options;
39+
40+
Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) {
41+
Aws::InitAPI(Options);
42+
}
43+
44+
Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) {
45+
Aws::ShutdownAPI(Options);
46+
}
47+
3548
static const std::vector<NArrow::NTest::TTestColumn> testYdbSchema = TTestSchema::YdbSchema();
3649
static const std::vector<NArrow::NTest::TTestColumn> testYdbPk = TTestSchema::YdbPkSchema();
3750

‎ydb/core/tx/columnshard/ut_schema/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ PEERDIR(
1818
library/cpp/getopt
1919
library/cpp/regex/pcre
2020
library/cpp/svnversion
21+
contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core
2122
ydb/core/testlib/default
2223
ydb/core/tx/columnshard/hooks/abstract
2324
ydb/core/tx/columnshard/hooks/testing

‎ydb/core/tx/datashard/import_s3.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include <ydb/core/tablet/resource_broker.h>
1515
#include <ydb/core/wrappers/s3_wrapper.h>
1616
#include <ydb/core/wrappers/s3_storage.h>
17+
#include <ydb/core/wrappers/s3_storage_config.h>
1718
#include <ydb/core/io_formats/ydb_dump/csv_ydb_dump.h>
1819
#include <ydb/public/lib/scheme_types/scheme_type_id.h>
1920

‎ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,27 @@
77
#include <util/string/cast.h>
88
#include <util/string/printf.h>
99

10+
#include <library/cpp/testing/hook/hook.h>
11+
12+
#include <aws/core/Aws.h>
13+
1014
using namespace NSchemeShardUT_Private;
1115
using namespace NKikimr::NWrappers::NTestHelpers;
1216

17+
namespace {
18+
19+
Aws::SDKOptions Options;
20+
21+
Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) {
22+
Aws::InitAPI(Options);
23+
}
24+
25+
Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) {
26+
Aws::ShutdownAPI(Options);
27+
}
28+
29+
}
30+
1331
Y_UNIT_TEST_SUITE(TBackupTests) {
1432
using TFillFn = std::function<void(TTestBasicRuntime&)>;
1533

‎ydb/core/tx/schemeshard/ut_backup/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ IF (NOT OS_WINDOWS)
2020
library/cpp/getopt
2121
library/cpp/regex/pcre
2222
library/cpp/svnversion
23+
contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core
2324
ydb/core/testlib/default
2425
ydb/core/tx
2526
ydb/core/tx/schemeshard/ut_helpers

‎ydb/core/tx/schemeshard/ut_export/ut_export.cpp

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,27 @@
1313
#include <util/string/printf.h>
1414
#include <util/system/env.h>
1515

16+
#include <library/cpp/testing/hook/hook.h>
17+
18+
#include <aws/core/Aws.h>
19+
1620
using namespace NSchemeShardUT_Private;
1721
using namespace NKikimr::NWrappers::NTestHelpers;
1822

1923
using TTablesWithAttrs = TVector<std::pair<TString, TMap<TString, TString>>>;
2024

2125
namespace {
2226

27+
Aws::SDKOptions Options;
28+
29+
Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) {
30+
Aws::InitAPI(Options);
31+
}
32+
33+
Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) {
34+
Aws::ShutdownAPI(Options);
35+
}
36+
2337
void Run(TTestBasicRuntime& runtime, TTestEnv& env, const std::variant<TVector<TString>, TTablesWithAttrs>& tablesVar, const TString& request,
2438
Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS,
2539
const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "", const TString& peerName = "") {
@@ -1846,7 +1860,7 @@ partitioning_settings {
18461860
return ev->Get<TEvSchemeShard::TEvModifySchemeTransaction>()->Record
18471861
.GetTransaction(0).GetOperationType() == NKikimrSchemeOp::ESchemeOpBackup;
18481862
};
1849-
1863+
18501864
THolder<IEventHandle> delayed;
18511865
auto prevObserver = runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& ev) {
18521866
if (delayFunc(ev)) {
@@ -2242,7 +2256,7 @@ partitioning_settings {
22422256
min_partitions_count: 10
22432257
)"));
22442258
}
2245-
2259+
22462260
Y_UNIT_TEST(UserSID) {
22472261
TTestBasicRuntime runtime;
22482262
TTestEnv env(runtime);

‎ydb/core/tx/schemeshard/ut_export/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ IF (NOT OS_WINDOWS)
2020
library/cpp/getopt
2121
library/cpp/regex/pcre
2222
library/cpp/svnversion
23+
contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core
2324
ydb/core/testlib/default
2425
ydb/core/tx
2526
ydb/core/tx/schemeshard/ut_helpers

‎ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,28 @@
44

55
#include <util/string/printf.h>
66

7+
#include <library/cpp/testing/hook/hook.h>
8+
9+
#include <aws/core/Aws.h>
10+
711
using namespace NSchemeShardUT_Private;
812
using namespace NSchemeShardUT_Private::NExportReboots;
913
using namespace NKikimr::NWrappers::NTestHelpers;
1014

15+
namespace {
16+
17+
Aws::SDKOptions Options;
18+
19+
Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) {
20+
Aws::InitAPI(Options);
21+
}
22+
23+
Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) {
24+
Aws::ShutdownAPI(Options);
25+
}
26+
27+
}
28+
1129
Y_UNIT_TEST_SUITE(TExportToS3WithRebootsTests) {
1230
using TUnderlying = std::function<void(const TVector<TString>&, const TString&, TTestWithReboots&)>;
1331

‎ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ PEERDIR(
1919
library/cpp/getopt
2020
library/cpp/regex/pcre
2121
library/cpp/svnversion
22+
contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core
2223
ydb/core/testlib/default
2324
ydb/core/tx
2425
ydb/core/tx/schemeshard/ut_helpers

‎ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@
2121

2222
#include <ydb/public/api/protos/ydb_import.pb.h>
2323

24+
#include <aws/core/Aws.h>
2425
#include <contrib/libs/zstd/include/zstd.h>
2526
#include <library/cpp/string_utils/quote/quote.h>
27+
#include <library/cpp/testing/hook/hook.h>
2628

2729
#include <util/datetime/base.h>
2830
#include <util/generic/size_literals.h>
@@ -38,6 +40,16 @@ using namespace NKikimr::NWrappers::NTestHelpers;
3840

3941
namespace {
4042

43+
Aws::SDKOptions Options;
44+
45+
Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) {
46+
Aws::InitAPI(Options);
47+
}
48+
49+
Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) {
50+
Aws::ShutdownAPI(Options);
51+
}
52+
4153
const TString EmptyYsonStr = R"([[[[];%false]]])";
4254

4355
TString GenerateScheme(const NKikimrSchemeOp::TPathDescription& pathDesc) {
@@ -317,7 +329,6 @@ namespace {
317329
runtime.SetObserverFunc(prevObserver);
318330
}
319331

320-
321332
} // anonymous
322333

323334
Y_UNIT_TEST_SUITE(TRestoreTests) {

‎ydb/core/tx/schemeshard/ut_restore/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ ELSE()
1414
ENDIF()
1515

1616
PEERDIR(
17+
contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core
1718
contrib/libs/double-conversion
1819
library/cpp/string_utils/quote
1920
ydb/core/kqp/ut/common

‎ydb/core/wrappers/s3_storage.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
#ifndef KIKIMR_DISABLE_S3_OPS
44

55
#include "abstract.h"
6-
#include "s3_storage_config.h"
76

87
#include <ydb/core/protos/flat_scheme_op.pb.h>
98
#include <ydb/core/wrappers/events/common.h>
@@ -32,7 +31,7 @@
3231

3332
namespace NKikimr::NWrappers::NExternalStorage {
3433

35-
class TS3ExternalStorage: public IExternalStorageOperator, TS3User {
34+
class TS3ExternalStorage: public IExternalStorageOperator {
3635
private:
3736
THolder<Aws::S3::S3Client> Client;
3837
const Aws::Client::ClientConfiguration Config;

0 commit comments

Comments
 (0)