Skip to content

Commit e4cd492

Browse files
swalrus1swalrus1
authored
Merge 442dc83 into 030f818 
2 parents 030f818 + 442dc83 commit e4cd492

File tree

6 files changed

+203
-29
lines changed

6 files changed

+203
-29
lines changed

‎ydb/core/tx/schemeshard/operations/metadata/tiering_rule/object.h‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#pragma once
2+
3+
#include <ydb/core/protos/flat_scheme_op.pb.h>
4+
#include <ydb/core/tx/schemeshard/operations/metadata/abstract/object.h>
5+
#include <ydb/core/tx/schemeshard/schemeshard_info_types.h>
6+
7+
namespace NKikimr::NSchemeShard::NOperations {
8+
9+
class TTieringRuleEntity: public TMetadataEntity {
10+
private:
11+
static TFactory::TRegistrator<TTieringRuleEntity> Registrator;
12+
13+
private:
14+
using TBase = TMetadataEntity;
15+
YDB_READONLY_DEF(TTieringRuleInfo::TPtr, TieringRuleInfo);
16+
17+
std::shared_ptr<TMetadataUpdateDrop> MakeDropUpdate() const override;
18+
19+
protected:
20+
[[nodiscard]] TConclusionStatus DoInitialize(const TEntityInitializationContext& context) override;
21+
22+
TTieringRuleEntity(const TPathId& pathId, const TTieringRuleInfo::TPtr& objectInfo)
23+
: TBase(pathId)
24+
, TieringRuleInfo(objectInfo) {
25+
}
26+
27+
public:
28+
TString GetClassName() const override {
29+
return "TIERING_RULE";
30+
}
31+
32+
public:
33+
TTieringRuleEntity(const TPathId& pathId)
34+
: TBase(pathId) {
35+
}
36+
};
37+
}

‎ydb/library/binary_json/benchmark/read.cpp‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#include <benchmark/benchmark.h>
2+
3+
#include <util/random/random.h>
4+
#include <library/cpp/testing/unittest/registar.h>
5+
#include <library/cpp/json/json_value.h>
6+
#include <library/cpp/json/json_writer.h>
7+
8+
#include <ydb/library/binary_json/write.h>
9+
10+
// ya test -r -D BENCHMARK_MAKE_LARGE_PART
11+
#ifndef BENCHMARK_MAKE_LARGE_PART
12+
#define BENCHMARK_MAKE_LARGE_PART 0
13+
#endif
14+
15+
using namespace NKikimr::NBinaryJson;
16+
17+
namespace {
18+
19+
static ui64 seed = 0;
20+
21+
NJson::TJsonValue GetTestJson(ui64 depth = 10, ui64 nChildren = 2) {
22+
NJson::TJsonValue value;
23+
if (depth == 1) {
24+
value.SetValue(NUnitTest::RandomString(10, seed++));
25+
return value;
26+
}
27+
for (ui64 i = 0; i < nChildren; ++i) {
28+
value.InsertValue(NUnitTest::RandomString(10, seed++), GetTestJson(depth - 1));
29+
}
30+
return value;
31+
}
32+
33+
TString GetTestJsonString() {
34+
seed = 42;
35+
return NJson::WriteJson(GetTestJson(2, 100));
36+
}
37+
38+
static void BenchSimdJson(benchmark::State& state) {
39+
TString value = GetTestJsonString();
40+
TStringBuf buf(value);
41+
for (auto _ : state) {
42+
auto result = SerializeToBinaryJson(buf);
43+
benchmark::DoNotOptimize(result);
44+
benchmark::ClobberMemory();
45+
}
46+
}
47+
48+
}
49+
50+
BENCHMARK(BenchWriteSimdJson);

‎ydb/library/binary_json/benchmark/ya.make‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
G_BENCHMARK()
2+
3+
TAG(ya:fat)
4+
SIZE(LARGE)
5+
TIMEOUT(600)
6+
7+
IF (BENCHMARK_MAKE_LARGE_PART)
8+
CFLAGS(
9+
-DBENCHMARK_MAKE_LARGE_PART=1
10+
)
11+
TIMEOUT(1200)
12+
ENDIF()
13+
14+
SRCS(
15+
read.cpp
16+
)
17+
18+
PEERDIR(
19+
library/cpp/testing/unittest
20+
ydb/library/binary_json
21+
)
22+
23+
YQL_LAST_ABI_VERSION()
24+
25+
END()

‎ydb/library/binary_json/write.cpp‎

Lines changed: 87 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
#include "write.h"
22

3+
#include <contrib/libs/simdjson/include/simdjson/dom/array-inl.h>
4+
#include <contrib/libs/simdjson/include/simdjson/dom/document-inl.h>
5+
#include <contrib/libs/simdjson/include/simdjson/dom/element-inl.h>
6+
#include <contrib/libs/simdjson/include/simdjson/dom/object-inl.h>
7+
#include <contrib/libs/simdjson/include/simdjson/dom/parser-inl.h>
38
#include <library/cpp/json/json_reader.h>
4-
5-
#include <util/generic/vector.h>
6-
#include <util/generic/stack.h>
7-
#include <util/generic/set.h>
89
#include <util/generic/algorithm.h>
910
#include <util/generic/map.h>
11+
#include <util/generic/set.h>
12+
#include <util/generic/stack.h>
13+
#include <util/generic/vector.h>
1014

1115
#include <cmath>
1216

@@ -74,38 +78,29 @@ struct TJsonIndex {
7478
ui32 InternKey(const TStringBuf value) {
7579
TotalKeysCount++;
7680

77-
const auto it = Keys.find(value);
78-
if (it == Keys.end()) {
79-
const ui32 currentIndex = LastFreeStringIndex++;
80-
Keys[TString(value)] = currentIndex;
81+
const auto [it, emplaced] = Keys.emplace(value, LastFreeStringIndex);
82+
if (emplaced) {
83+
++LastFreeStringIndex;
8184
TotalKeyLength += value.length() + 1;
82-
return currentIndex;
83-
} else {
84-
return it->second;
8585
}
86+
return it->second;
8687
}
8788

8889
ui32 InternString(const TStringBuf value) {
89-
const auto it = Strings.find(value);
90-
if (it == Strings.end()) {
91-
const ui32 currentIndex = LastFreeStringIndex++;
92-
Strings[value] = currentIndex;
90+
const auto [it, emplaced] = Keys.emplace(value, LastFreeStringIndex);
91+
if (emplaced) {
92+
++LastFreeStringIndex;
9393
TotalStringLength += value.length() + 1;
94-
return currentIndex;
95-
} else {
96-
return it->second;
9794
}
95+
return it->second;
9896
}
9997

10098
ui32 InternNumber(double value) {
101-
const auto it = Numbers.find(value);
102-
if (it == Numbers.end()) {
103-
const ui32 currentIndex = LastFreeNumberIndex++;
104-
Numbers[value] = currentIndex;
105-
return currentIndex;
106-
} else {
107-
return it->second;
99+
const auto [it, emplaced] = Numbers.emplace(value, LastFreeNumberIndex);
100+
if (emplaced) {
101+
++LastFreeNumberIndex;
108102
}
103+
return it->second;
109104
}
110105

111106
void AddContainer(EContainerType type) {
@@ -551,17 +546,80 @@ void DomToJsonIndex(const NUdf::TUnboxedValue& value, TBinaryJsonCallbacks& call
551546
}
552547
}
553548

549+
void SimdJsonToJsonIndex(const simdjson::dom::element& value, TBinaryJsonCallbacks& callbacks) {
550+
switch (value.type()) {
551+
case simdjson::dom::element_type::STRING: {
552+
std::string_view v;
553+
Y_ABORT_UNLESS(value.get(v) == simdjson::SUCCESS);
554+
callbacks.OnString(v);
555+
break;
556+
}
557+
case simdjson::dom::element_type::BOOL: {
558+
bool v;
559+
Y_ABORT_UNLESS(value.get(v) == simdjson::SUCCESS);
560+
callbacks.OnBoolean(v);
561+
break;
562+
}
563+
case simdjson::dom::element_type::INT64: {
564+
i64 v;
565+
Y_ABORT_UNLESS(value.get(v) == simdjson::SUCCESS);
566+
callbacks.OnInteger(v);
567+
break;
568+
}
569+
case simdjson::dom::element_type::UINT64: {
570+
ui64 v;
571+
Y_ABORT_UNLESS(value.get(v) == simdjson::SUCCESS);
572+
callbacks.OnUInteger(v);
573+
break;
574+
}
575+
case simdjson::dom::element_type::DOUBLE: {
576+
double v;
577+
Y_ABORT_UNLESS(value.get(v) == simdjson::SUCCESS);
578+
callbacks.OnDouble(v);
579+
break;
580+
}
581+
case simdjson::dom::element_type::NULL_VALUE:
582+
callbacks.OnNull();
583+
break;
584+
case simdjson::dom::element_type::ARRAY: {
585+
callbacks.OnOpenArray();
586+
587+
simdjson::dom::array v;
588+
Y_ABORT_UNLESS(value.get(v) == simdjson::SUCCESS);
589+
for (const auto& item : v) {
590+
SimdJsonToJsonIndex(item, callbacks);
591+
}
592+
593+
callbacks.OnCloseArray();
594+
break;
595+
}
596+
case simdjson::dom::element_type::OBJECT: {
597+
callbacks.OnOpenMap();
598+
599+
simdjson::dom::object v;
600+
Y_ABORT_UNLESS(value.get(v) == simdjson::SUCCESS);
601+
for (const auto& item : v) {
602+
callbacks.OnMapKey(item.key);
603+
SimdJsonToJsonIndex(item.value, callbacks);
604+
}
605+
606+
callbacks.OnCloseMap();
607+
break;
608+
}
609+
}
610+
}
554611
}
555612

556613
TMaybe<TBinaryJson> SerializeToBinaryJsonImpl(const TStringBuf json) {
557-
TMemoryInput input(json.data(), json.size());
558-
TBinaryJsonCallbacks callbacks(/* throwException */ false);
559-
if (!ReadJson(&input, &callbacks)) {
614+
simdjson::dom::parser parser;
615+
auto doc = parser.parse(json);
616+
if (doc.error() != simdjson::SUCCESS) {
560617
return Nothing();
561618
}
619+
TBinaryJsonCallbacks callbacks(/* throwException */ false);
620+
SimdJsonToJsonIndex(doc.value(), callbacks);
562621
TBinaryJsonSerializer serializer(std::move(callbacks).GetResult());
563622
return std::move(serializer).Serialize();
564-
565623
}
566624

567625
TMaybe<TBinaryJson> SerializeToBinaryJson(const TStringBuf json) {

‎ydb/library/binary_json/write.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ namespace NKikimr::NBinaryJson {
1212
* @brief Translates textual JSON into BinaryJson
1313
*/
1414
TMaybe<TBinaryJson> SerializeToBinaryJson(const TStringBuf json);
15+
TMaybe<TBinaryJson> SerializeToBinaryJsonOndemand(const TStringBuf json);
16+
TMaybe<TBinaryJson> SerializeToBinaryJsonRapidjson(const TStringBuf json);
1517

1618
/**
1719
* @brief Translates DOM layout from `yql/library/dom` library into BinaryJson

‎ydb/library/binary_json/ya.make‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ YQL_ABI_VERSION(
99
PEERDIR(
1010
library/cpp/json
1111
ydb/library/yql/minikql/dom
12+
contrib/libs/simdjson
1213
)
1314

1415
SRCS(
@@ -23,4 +24,5 @@ END()
2324

2425
RECURSE_FOR_TESTS(
2526
ut
27+
benchmark
2628
)

0 commit comments

Comments
 (0)