Skip to content

Commit 63f8217

Browse files
pixccpixcc
authored
Merge 179ba6a into dbeed94 
2 parents dbeed94 + 179ba6a commit 63f8217

File tree

10 files changed

+216
-3
lines changed

10 files changed

+216
-3
lines changed

‎ydb/core/protos/export.proto‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ import "ydb/public/api/protos/ydb_issue_message.proto";
33
import "ydb/public/api/protos/ydb_operation.proto";
44
import "ydb/public/api/protos/ydb_status_codes.proto";
55

6+
import "google/protobuf/timestamp.proto";
7+
68
package NKikimrExport;
79
option java_package = "ru.yandex.kikimr.proto";
810

@@ -11,6 +13,8 @@ message TExport {
1113
optional Ydb.StatusIds.StatusCode Status = 2;
1214
repeated Ydb.Issue.IssueMessage Issues = 3;
1315
optional Ydb.Export.ExportProgress.Progress Progress = 4;
16+
optional google.protobuf.Timestamp StartTime = 8;
17+
optional google.protobuf.Timestamp EndTime = 9;
1418
repeated Ydb.Export.ExportItemProgress ItemsProgress = 7;
1519
oneof Settings {
1620
Ydb.Export.ExportToYtSettings ExportToYtSettings = 5;

‎ydb/core/tx/schemeshard/schemeshard__init.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4190,6 +4190,9 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
41904190
exportInfo->WaitTxId = rowset.GetValueOrDefault<Schema::Exports::WaitTxId>(InvalidTxId);
41914191
exportInfo->Issue = rowset.GetValueOrDefault<Schema::Exports::Issue>(TString());
41924192

4193+
exportInfo->StartTime = TInstant::Seconds(rowset.GetValueOrDefault<Schema::Exports::StartTime>());
4194+
exportInfo->EndTime = TInstant::Seconds(rowset.GetValueOrDefault<Schema::Exports::EndTime>());
4195+
41934196
Self->Exports[id] = exportInfo;
41944197
if (uid) {
41954198
Self->ExportsByUid[uid] = exportInfo;

‎ydb/core/tx/schemeshard/schemeshard_export.cpp‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,11 @@ namespace {
8383
void TSchemeShard::FromXxportInfo(NKikimrExport::TExport& exprt, const TExportInfo::TPtr exportInfo) {
8484
exprt.SetId(exportInfo->Id);
8585
exprt.SetStatus(Ydb::StatusIds::SUCCESS);
86+
87+
*exprt.MutableStartTime() = SecondsToProtoTimeStamp(exportInfo->StartTime.Seconds());
88+
if (exportInfo->EndTime != TInstant::Zero()) {
89+
*exprt.MutableEndTime() = SecondsToProtoTimeStamp(exportInfo->EndTime.Seconds());
90+
}
8691

8792
switch (exportInfo->State) {
8893
case TExportInfo::EState::CreateExportDir:
@@ -182,7 +187,9 @@ void TSchemeShard::PersistExportState(NIceDb::TNiceDb& db, const TExportInfo::TP
182187
db.Table<Schema::Exports>().Key(exportInfo->Id).Update(
183188
NIceDb::TUpdate<Schema::Exports::State>(static_cast<ui8>(exportInfo->State)),
184189
NIceDb::TUpdate<Schema::Exports::WaitTxId>(exportInfo->WaitTxId),
185-
NIceDb::TUpdate<Schema::Exports::Issue>(exportInfo->Issue)
190+
NIceDb::TUpdate<Schema::Exports::Issue>(exportInfo->Issue),
191+
NIceDb::TUpdate<Schema::Exports::StartTime>(exportInfo->StartTime.Seconds()),
192+
NIceDb::TUpdate<Schema::Exports::EndTime>(exportInfo->EndTime.Seconds())
186193
);
187194
}
188195

‎ydb/core/tx/schemeshard/schemeshard_export__cancel.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,10 @@ struct TSchemeShard::TExport::TTxCancel: public TSchemeShard::TXxport::TTxBase {
7979
}
8080
}
8181

82+
if (exportInfo->State == TExportInfo::EState::Cancelled) {
83+
exportInfo->EndTime = TAppData::TimeProvider->Now();
84+
}
85+
8286
NIceDb::TNiceDb db(txc.DB);
8387
Self->PersistExportState(db, exportInfo);
8488

@@ -158,6 +162,7 @@ struct TSchemeShard::TExport::TTxCancelAck: public TSchemeShard::TXxport::TTxBas
158162

159163
if (cancelledItems == cancellableItems) {
160164
exportInfo->State = TExportInfo::EState::Cancelled;
165+
exportInfo->EndTime = TAppData::TimeProvider->Now();
161166
Self->PersistExportState(db, exportInfo);
162167
}
163168

‎ydb/core/tx/schemeshard/schemeshard_export__create.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ struct TSchemeShard::TExport::TTxCreate: public TSchemeShard::TXxport::TTxBase {
142142
Self->PersistCreateExport(db, exportInfo);
143143

144144
exportInfo->State = TExportInfo::EState::CreateExportDir;
145+
exportInfo->StartTime = TAppData::TimeProvider->Now();
145146
Self->PersistExportState(db, exportInfo);
146147

147148
Self->Exports[id] = exportInfo;
@@ -485,6 +486,10 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase
485486

486487
CancelTransferring(exportInfo, i);
487488
}
489+
490+
if (exportInfo->State == EState::Cancelled) {
491+
exportInfo->EndTime = TAppData::TimeProvider->Now();
492+
}
488493
}
489494

490495
TMaybe<TString> GetIssues(const TPathId& itemPathId, TTxId backupTxId) {
@@ -581,6 +586,10 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase
581586
}
582587
}
583588

589+
if (exportInfo->State == EState::Cancelled) {
590+
exportInfo->EndTime = TAppData::TimeProvider->Now();
591+
}
592+
584593
Self->PersistExportState(db, exportInfo);
585594
SendNotificationsIfFinished(exportInfo);
586595
break;
@@ -804,6 +813,7 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase
804813

805814
exportInfo->Issue = record.GetReason();
806815
exportInfo->State = EState::Cancelled;
816+
exportInfo->EndTime = TAppData::TimeProvider->Now();
807817
}
808818

809819
Self->PersistExportState(db, exportInfo);
@@ -948,6 +958,7 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase
948958
} else {
949959
if (AllOf(exportInfo->Items, &TExportInfo::TItem::IsDone)) {
950960
exportInfo->State = EState::Done;
961+
exportInfo->EndTime = TAppData::TimeProvider->Now();
951962
}
952963
}
953964

‎ydb/core/tx/schemeshard/schemeshard_info_types.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2593,6 +2593,9 @@ struct TExportInfo: public TSimpleRefCount<TExportInfo> {
25932593
ui64 SnapshotStep = 0;
25942594
ui64 SnapshotTxId = 0;
25952595

2596+
TInstant StartTime = TInstant::Zero();
2597+
TInstant EndTime = TInstant::Zero();
2598+
25962599
explicit TExportInfo(
25972600
const ui64 id,
25982601
const TString& uid,

‎ydb/core/tx/schemeshard/schemeshard_schema.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1151,6 +1151,9 @@ struct Schema : NIceDb::Schema {
11511151
struct ExportOwnerPathId : Column<10, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; };
11521152
struct DomainPathOwnerId : Column<11, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; };
11531153

1154+
struct StartTime : Column<14, NScheme::NTypeIds::Uint64> {};
1155+
struct EndTime : Column<15, NScheme::NTypeIds::Uint64> {};
1156+
11541157
using TKey = TableKey<Id>;
11551158
using TColumns = TableColumns<
11561159
Id,
@@ -1165,7 +1168,9 @@ struct Schema : NIceDb::Schema {
11651168
ExportOwnerPathId,
11661169
DomainPathOwnerId,
11671170
Kind,
1168-
UserSID
1171+
UserSID,
1172+
StartTime,
1173+
EndTime
11691174
>;
11701175
};
11711176

‎ydb/core/tx/schemeshard/ut_export/ut_export.cpp‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1503,4 +1503,162 @@ partitioning_settings {
15031503
TestGetExport(runtime, exportId, "/MyRoot");
15041504
env.TestWaitNotification(runtime, exportId);
15051505
}
1506+
1507+
Y_UNIT_TEST(ExportStartTime) {
1508+
TTestBasicRuntime runtime;
1509+
TTestEnv env(runtime);
1510+
ui64 txId = 100;
1511+
1512+
TestCreateTable(runtime, ++txId, "/MyRoot", R"(
1513+
Name: "Table"
1514+
Columns { Name: "key" Type: "Utf8" }
1515+
Columns { Name: "value" Type: "Utf8" }
1516+
KeyColumnNames: ["key"]
1517+
)");
1518+
env.TestWaitNotification(runtime, txId);
1519+
1520+
TPortManager portManager;
1521+
const ui16 port = portManager.GetPort();
1522+
1523+
TS3Mock s3Mock({}, TS3Mock::TSettings(port));
1524+
UNIT_ASSERT(s3Mock.Start());
1525+
1526+
TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"(
1527+
ExportToS3Settings {
1528+
endpoint: "localhost:%d"
1529+
scheme: HTTP
1530+
items {
1531+
source_path: "/MyRoot/Table"
1532+
destination_prefix: ""
1533+
}
1534+
}
1535+
)", port));
1536+
1537+
const auto desc = TestGetExport(runtime, txId, "/MyRoot");
1538+
const auto& entry = desc.GetResponse().GetEntry();
1539+
UNIT_ASSERT_VALUES_EQUAL(entry.GetProgress(), Ydb::Export::ExportProgress::PROGRESS_PREPARING);
1540+
UNIT_ASSERT(entry.HasStartTime());
1541+
UNIT_ASSERT(!entry.HasEndTime());
1542+
}
1543+
1544+
Y_UNIT_TEST(CompletedExportEndTime) {
1545+
TTestBasicRuntime runtime;
1546+
TTestEnv env(runtime);
1547+
ui64 txId = 100;
1548+
1549+
TestCreateTable(runtime, ++txId, "/MyRoot", R"(
1550+
Name: "Table"
1551+
Columns { Name: "key" Type: "Utf8" }
1552+
Columns { Name: "value" Type: "Utf8" }
1553+
KeyColumnNames: ["key"]
1554+
)");
1555+
env.TestWaitNotification(runtime, txId);
1556+
1557+
TPortManager portManager;
1558+
const ui16 port = portManager.GetPort();
1559+
1560+
TS3Mock s3Mock({}, TS3Mock::TSettings(port));
1561+
UNIT_ASSERT(s3Mock.Start());
1562+
1563+
TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"(
1564+
ExportToS3Settings {
1565+
endpoint: "localhost:%d"
1566+
scheme: HTTP
1567+
items {
1568+
source_path: "/MyRoot/Table"
1569+
destination_prefix: ""
1570+
}
1571+
}
1572+
)", port));
1573+
1574+
runtime.AdvanceCurrentTime(TDuration::Seconds(30)); // doing backup
1575+
1576+
env.TestWaitNotification(runtime, txId);
1577+
1578+
const auto desc = TestGetExport(runtime, txId, "/MyRoot");
1579+
const auto& entry = desc.GetResponse().GetEntry();
1580+
UNIT_ASSERT_VALUES_EQUAL(entry.GetProgress(), Ydb::Export::ExportProgress::PROGRESS_DONE);
1581+
UNIT_ASSERT(entry.HasStartTime());
1582+
UNIT_ASSERT(entry.HasEndTime());
1583+
UNIT_ASSERT_LT(entry.GetStartTime().seconds(), entry.GetEndTime().seconds());
1584+
}
1585+
1586+
Y_UNIT_TEST(CancelledExportEndTime) {
1587+
TTestBasicRuntime runtime;
1588+
TTestEnv env(runtime);
1589+
ui64 txId = 100;
1590+
1591+
TestCreateTable(runtime, ++txId, "/MyRoot", R"(
1592+
Name: "Table"
1593+
Columns { Name: "key" Type: "Utf8" }
1594+
Columns { Name: "value" Type: "Utf8" }
1595+
KeyColumnNames: ["key"]
1596+
)");
1597+
env.TestWaitNotification(runtime, txId);
1598+
1599+
auto delayFunc = [](TAutoPtr<IEventHandle>& ev) {
1600+
if (ev->GetTypeRewrite() != TEvSchemeShard::EvModifySchemeTransaction) {
1601+
return false;
1602+
}
1603+
1604+
return ev->Get<TEvSchemeShard::TEvModifySchemeTransaction>()->Record
1605+
.GetTransaction(0).GetOperationType() == NKikimrSchemeOp::ESchemeOpBackup;
1606+
};
1607+
1608+
THolder<IEventHandle> delayed;
1609+
auto prevObserver = runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& ev) {
1610+
if (delayFunc(ev)) {
1611+
delayed.Reset(ev.Release());
1612+
return TTestActorRuntime::EEventAction::DROP;
1613+
}
1614+
return TTestActorRuntime::EEventAction::PROCESS;
1615+
});
1616+
1617+
TPortManager portManager;
1618+
const ui16 port = portManager.GetPort();
1619+
1620+
TS3Mock s3Mock({}, TS3Mock::TSettings(port));
1621+
UNIT_ASSERT(s3Mock.Start());
1622+
1623+
TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"(
1624+
ExportToS3Settings {
1625+
endpoint: "localhost:%d"
1626+
scheme: HTTP
1627+
items {
1628+
source_path: "/MyRoot/Table"
1629+
destination_prefix: ""
1630+
}
1631+
}
1632+
)", port));
1633+
const ui64 exportId = txId;
1634+
1635+
runtime.AdvanceCurrentTime(TDuration::Seconds(30)); // doing backup
1636+
1637+
if (!delayed) {
1638+
TDispatchOptions opts;
1639+
opts.FinalEvents.emplace_back([&delayed](IEventHandle&) -> bool {
1640+
return bool(delayed);
1641+
});
1642+
runtime.DispatchEvents(opts);
1643+
}
1644+
runtime.SetObserverFunc(prevObserver);
1645+
1646+
TestCancelExport(runtime, ++txId, "/MyRoot", exportId);
1647+
1648+
auto desc = TestGetExport(runtime, exportId, "/MyRoot");
1649+
auto entry = desc.GetResponse().GetEntry();
1650+
UNIT_ASSERT_VALUES_EQUAL(entry.GetProgress(), Ydb::Export::ExportProgress::PROGRESS_CANCELLATION);
1651+
UNIT_ASSERT(entry.HasStartTime());
1652+
UNIT_ASSERT(!entry.HasEndTime());
1653+
1654+
runtime.Send(delayed.Release(), 0, true);
1655+
env.TestWaitNotification(runtime, exportId);
1656+
1657+
desc = TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::CANCELLED);
1658+
entry = desc.GetResponse().GetEntry();
1659+
UNIT_ASSERT_VALUES_EQUAL(entry.GetProgress(), Ydb::Export::ExportProgress::PROGRESS_CANCELLED);
1660+
UNIT_ASSERT(entry.HasStartTime());
1661+
UNIT_ASSERT(entry.HasEndTime());
1662+
UNIT_ASSERT_LT(entry.GetStartTime().seconds(), entry.GetEndTime().seconds());
1663+
}
15061664
}

‎ydb/public/api/protos/out/out.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include <ydb/public/api/protos/ydb_cms.pb.h>
22
#include <ydb/public/api/protos/ydb_monitoring.pb.h>
33
#include <ydb/public/api/protos/ydb_status_codes.pb.h>
4+
#include <ydb/public/api/protos/ydb_export.pb.h>
45

56
#include <util/stream/output.h>
67

@@ -19,3 +20,7 @@ Y_DECLARE_OUT_SPEC(, Ydb::Monitoring::SelfCheck::Result, stream, value) {
1920
Y_DECLARE_OUT_SPEC(, Ydb::Monitoring::StatusFlag::Status, stream, value) {
2021
stream << Ydb::Monitoring::StatusFlag_Status_Name(value);
2122
}
23+
24+
Y_DECLARE_OUT_SPEC(, Ydb::Export::ExportProgress::Progress, stream, value) {
25+
stream << Ydb::Export::ExportProgress_Progress_Name(value);
26+
}

‎ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2936,6 +2936,16 @@
29362936
"ColumnId": 13,
29372937
"ColumnName": "UserSID",
29382938
"ColumnType": "Utf8"
2939+
},
2940+
{
2941+
"ColumnId": 14,
2942+
"ColumnName": "StartTime",
2943+
"ColumnType": "Uint64"
2944+
},
2945+
{
2946+
"ColumnId": 15,
2947+
"ColumnName": "EndTime",
2948+
"ColumnType": "Uint64"
29392949
}
29402950
],
29412951
"ColumnsDropped": [],
@@ -2954,7 +2964,9 @@
29542964
10,
29552965
11,
29562966
12,
2957-
13
2967+
13,
2968+
14,
2969+
15
29582970
],
29592971
"RoomID": 0,
29602972
"Codec": 0,

0 commit comments

Comments
 (0)