Skip to content

Commit 5f12295

Browse files
SammyVimesSammyVimes
authored
YDB-2757 Fix UUID column export (#2809)
1 parent 2536bf1 commit 5f12295

File tree

6 files changed

+135
-1
lines changed

6 files changed

+135
-1
lines changed

‎ydb/core/io_formats/cell_maker/cell_maker.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include <ydb/library/binary_json/write.h>
44
#include <ydb/library/dynumber/dynumber.h>
5+
#include <ydb/library/uuid/uuid.h>
56

67
#include <ydb/library/yql/minikql/dom/yson.h>
78
#include <ydb/library/yql/minikql/dom/json.h>
@@ -132,6 +133,21 @@ namespace {
132133
return true;
133134
}
134135

136+
struct TUuidHolder {
137+
union {
138+
ui16 Array[8];
139+
ui64 Halves[2];
140+
} Buf;
141+
};
142+
143+
template <>
144+
bool TryParse(TStringBuf value, TUuidHolder& result) {
145+
if (!NUuid::ParseUuidToArray(value, result.Buf.Array, false)) {
146+
return false;
147+
}
148+
return true;
149+
}
150+
135151
template <typename T, typename U>
136152
using TConverter = std::function<U(const T&)>;
137153

@@ -171,6 +187,14 @@ namespace {
171187
return v.Str;
172188
}
173189

190+
TStringBuf UuidToStringBuf(const TUuidHolder& uuid) {
191+
char uuidBuf[16];
192+
193+
NUuid::UuidHalfsToBytes(uuidBuf, 16, uuid.Buf.Halves[1], uuid.Buf.Halves[0]);
194+
195+
return TStringBuf(uuidBuf, 16);
196+
}
197+
174198
template <typename T, typename U = T>
175199
struct TCellMaker {
176200
static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TString& err, TConverter<T, U> conv = &Implicit<T, U>) {
@@ -297,6 +321,8 @@ bool MakeCell(TCell& cell, TStringBuf value, NScheme::TTypeInfo type, TMemoryPoo
297321
return TCellMaker<NYql::NDecimal::TInt128, std::pair<ui64, ui64>>::Make(cell, value, pool, err, &Int128ToPair);
298322
case NScheme::NTypeIds::Pg:
299323
return TCellMaker<NPg::TConvertResult, TStringBuf>::Make(cell, value, pool, err, &PgToStringBuf, type.GetTypeDesc());
324+
case NScheme::NTypeIds::Uuid:
325+
return TCellMaker<TUuidHolder, TStringBuf>::Make(cell, value, pool, err, &UuidToStringBuf);
300326
default:
301327
return false;
302328
}
@@ -390,6 +416,7 @@ bool CheckCellValue(const TCell& cell, NScheme::TTypeInfo type) {
390416
case NScheme::NTypeIds::JsonDocument: // checked at parsing time
391417
case NScheme::NTypeIds::DyNumber: // checked at parsing time
392418
case NScheme::NTypeIds::Pg: // checked at parsing time
419+
case NScheme::NTypeIds::Uuid: // checked at parsing time
393420
return true;
394421
case NScheme::NTypeIds::Date:
395422
return cell.AsValue<ui16>() < NUdf::MAX_DATE;

‎ydb/core/tx/datashard/export_common.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,5 +117,15 @@ bool PgToStream(TStringBuf data, void* typeDesc, IOutputStream& out, TString& er
117117
return true;
118118
}
119119

120+
bool UuidToStream(const std::pair<ui64, ui64>& loHi, IOutputStream& out, TString& err) {
121+
Y_UNUSED(err);
122+
123+
NYdb::TUuidValue uuid(loHi.first, loHi.second);
124+
125+
out << uuid.ToString();
126+
127+
return true;
128+
}
129+
120130
} // NDataShard
121131
} // NKikimr

‎ydb/core/tx/datashard/export_common.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ TString DyNumberToString(TStringBuf data);
3939
bool DecimalToStream(const std::pair<ui64, i64>& loHi, IOutputStream& out, TString& err);
4040
bool DyNumberToStream(TStringBuf data, IOutputStream& out, TString& err);
4141
bool PgToStream(TStringBuf data, void* typeDesc, IOutputStream& out, TString& err);
42+
bool UuidToStream(const std::pair<ui64, ui64>& loHi, IOutputStream& out, TString& err);
4243

4344
} // NDataShard
4445
} // NKikimr

‎ydb/core/tx/datashard/export_s3_buffer_raw.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,9 @@ bool TS3BufferRaw::Collect(const NTable::IScan::TRow& row, IOutputStream& out) {
125125
case NScheme::NTypeIds::Pg:
126126
serialized = PgToStream(cell.AsBuf(), column.Type.GetTypeDesc(), out, ErrorString);
127127
break;
128+
case NScheme::NTypeIds::Uuid:
129+
serialized = UuidToStream(cell.AsValue<std::pair<ui64, ui64>>(), out, ErrorString);
130+
break;
128131
default:
129132
Y_ABORT("Unsupported type");
130133
}

‎ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,30 @@ Y_UNIT_TEST_SUITE(TBackupTests) {
100100
});
101101
}
102102

103+
Y_UNIT_TEST_WITH_COMPRESSION(BackupUuidColumn) {
104+
TTestBasicRuntime runtime;
105+
106+
Backup(runtime, ToString(Codec), R"(
107+
Name: "Table"
108+
Columns { Name: "key" Type: "Uint32" }
109+
Columns { Name: "value" Type: "Uuid" }
110+
KeyColumnNames: ["key"]
111+
)", [](TTestBasicRuntime& runtime) {
112+
NKikimrMiniKQL::TResult result;
113+
TString error;
114+
NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, TTestTxConfig::FakeHiveTablets, Sprintf(R"(
115+
(
116+
(let key '( '('key (Uint32 '%d) ) ) )
117+
(let row '( '('value (Uuid '"%s") ) ) )
118+
(return (AsList (UpdateRow '__user__%s key row) ))
119+
)
120+
)", 1, "0000111122223333", "Table"), result, error);
121+
122+
UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error);
123+
UNIT_ASSERT_VALUES_EQUAL(error, "");
124+
});
125+
}
126+
103127
template<ECompressionCodec Codec>
104128
void ShouldSucceedOnLargeData(ui32 minWriteBatchSize, const std::pair<ui32, ui32>& expectedResult) {
105129
TTestBasicRuntime runtime;

‎ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -911,7 +911,76 @@ value {
911911
env.TestWaitNotification(runtime, txId);
912912
TestGetExport(runtime, txId, "/MyRoot");
913913

914-
TestImport(runtime, txId, "/MyRoot", Sprintf(R"(
914+
TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"(
915+
ImportFromS3Settings {
916+
endpoint: "localhost:%d"
917+
scheme: HTTP
918+
items {
919+
source_prefix: "Backup1"
920+
destination_path: "/MyRoot/Restored"
921+
}
922+
}
923+
)", port));
924+
env.TestWaitNotification(runtime, txId);
925+
TestGetImport(runtime, txId, "/MyRoot");
926+
}
927+
928+
Y_UNIT_TEST(ExportImportUuid) {
929+
TTestBasicRuntime runtime;
930+
TTestEnv env(runtime, TTestEnvOptions().EnableTablePgTypes(true));
931+
ui64 txId = 100;
932+
933+
TestCreateTable(runtime, ++txId, "/MyRoot", R"(
934+
Name: "Table"
935+
Columns { Name: "key" Type: "Uint32" }
936+
Columns { Name: "value" Type: "Uuid" }
937+
KeyColumnNames: ["key"]
938+
)");
939+
env.TestWaitNotification(runtime, txId);
940+
941+
{
942+
TString tablePath = "/MyRoot/Table";
943+
int partitionIdx = 0;
944+
945+
auto tableDesc = DescribePath(runtime, tablePath, true, true);
946+
const auto& tablePartitions = tableDesc.GetPathDescription().GetTablePartitions();
947+
UNIT_ASSERT(partitionIdx < tablePartitions.size());
948+
const ui64 datashardTabletId = tablePartitions[partitionIdx].GetDatashardId();
949+
950+
NKikimrMiniKQL::TResult result;
951+
TString error;
952+
NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, datashardTabletId, Sprintf(R"(
953+
(
954+
(let key '( '('key (Uint32 '%d) ) ) )
955+
(let row '( '('value (Uuid '"%s") ) ) )
956+
(return (AsList (UpdateRow '__user__%s key row) ))
957+
)
958+
)", 1, "0123456789012345", "Table"), result, error);
959+
960+
UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, error);
961+
UNIT_ASSERT_VALUES_EQUAL(error, "");
962+
}
963+
964+
TPortManager portManager;
965+
const ui16 port = portManager.GetPort();
966+
967+
TS3Mock s3Mock({}, TS3Mock::TSettings(port));
968+
UNIT_ASSERT(s3Mock.Start());
969+
970+
TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"(
971+
ExportToS3Settings {
972+
endpoint: "localhost:%d"
973+
scheme: HTTP
974+
items {
975+
source_path: "/MyRoot/Table"
976+
destination_prefix: "Backup1"
977+
}
978+
}
979+
)", port));
980+
env.TestWaitNotification(runtime, txId);
981+
TestGetExport(runtime, txId, "/MyRoot");
982+
983+
TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"(
915984
ImportFromS3Settings {
916985
endpoint: "localhost:%d"
917986
scheme: HTTP

0 commit comments

Comments
 (0)