Skip to content

Commit bd74eca

Browse files
EmgarikoEmgariko
authored
Merge 60377df into 78c0da6 
2 parents 78c0da6 + 60377df commit bd74eca

File tree

4 files changed

+103
-39
lines changed

4 files changed

+103
-39
lines changed

‎ydb/library/workload/abstract/workload_query_generator.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ class TWorkloadParams {
179179
}
180180
virtual TString GetWorkloadName() const = 0;
181181

182+
virtual void Validate(const ECommandType /*commandType*/, int /*workloadType*/) const {};
182183
public:
183184
ui64 BulkSize = 10000;
184185
std::string DbPath;

‎ydb/library/workload/log/log.cpp‎

Lines changed: 93 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
#include <library/cpp/resource/resource.h>
55
#include <util/datetime/base.h>
66
#include <util/generic/guid.h>
7+
#include <util/random/entropy.h>
8+
#include <util/random/mersenne.h>
79
#include <util/random/normal.h>
810
#include <util/random/random.h>
911
#include <util/string/split.h>
@@ -252,6 +254,11 @@ class TRandomLogGenerator {
252254
return result.str();
253255
}
254256

257+
TInstant UniformInstant(ui64 from, ui64 to) const {
258+
TMersenne<ui64> rnd(Seed());
259+
return TInstant::FromValue(rnd.Uniform(from, to));
260+
}
261+
255262
TInstant RandomInstant() const {
256263
auto result = TInstant::Now() - TDuration::Seconds(Params.TimestampSubtract);
257264
i64 millisecondsDiff = 60 * 1000 * NormalRandom<double>(0., Params.TimestampStandardDeviationMinutes);
@@ -279,7 +286,7 @@ class TRandomLogGenerator {
279286
for (size_t row = 0; row < count; ++row) {
280287
result.emplace_back();
281288
result.back().LogId = CreateGuidAsString().c_str();
282-
result.back().Ts = RandomInstant();
289+
result.back().Ts = !!Params.TimestampDateFrom && !!Params.TimestampDateTo ? UniformInstant(*Params.TimestampDateFrom, *Params.TimestampDateTo) : RandomInstant();
283290
result.back().Level = RandomNumber<ui32>(10);
284291
result.back().ServiceName = RandomWord(false);
285292
result.back().Component = RandomWord(true);
@@ -360,6 +367,88 @@ TQueryInfoList TLogGenerator::GetWorkload(int type) {
360367
}
361368
}
362369

370+
void TLogWorkloadParams::ConfigureOptsColumns(NLastGetopt::TOpts& opts) {
371+
opts.AddLongOption("len", "String len")
372+
.DefaultValue(StringLen).StoreResult(&StringLen);
373+
opts.AddLongOption("int-cols", "Number of int columns")
374+
.DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt);
375+
opts.AddLongOption("str-cols", "Number of string columns")
376+
.DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt);
377+
opts.AddLongOption("key-cols", "Number of key columns")
378+
.DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt);
379+
}
380+
381+
void TLogWorkloadParams::ConfigureOptsFillData(NLastGetopt::TOpts& opts) {
382+
ConfigureOptsColumns(opts);
383+
opts.AddLongOption("rows", "Number of rows to upsert")
384+
.DefaultValue(RowsCnt).StoreResult(&RowsCnt);
385+
opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.")
386+
.DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes);
387+
opts.AddLongOption("date-from", "Left boundary of the interval to generate "
388+
"timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' "
389+
"should be passed as well. This option is mutually exclusive with 'timestamp_deviation'")
390+
.StoreResult(&TimestampDateFrom);
391+
opts.AddLongOption("date-to", "Right boundary of the interval to generate "
392+
"timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' "
393+
"should be passed as well. This option is mutually exclusive with 'timestamp_deviation'")
394+
.StoreResult(&TimestampDateTo);
395+
opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted")
396+
.DefaultValue(0).StoreResult(&TimestampSubtract);
397+
opts.AddLongOption("null-percent", "Percent of nulls in generated data")
398+
.DefaultValue(NullPercent).StoreResult(&NullPercent);
399+
}
400+
401+
void TLogWorkloadParams::Validate(const ECommandType commandType, int workloadType) const {
402+
const bool timestampDevPassed = TimestampStandardDeviationMinutes;
403+
const bool dateFromPassed = !!TimestampDateFrom;
404+
const bool dateToPassed = !!TimestampDateTo;
405+
406+
switch (commandType) {
407+
case TWorkloadParams::ECommandType::Init:
408+
break;
409+
case TWorkloadParams::ECommandType::Run:
410+
switch (static_cast<TLogGenerator::EType>(workloadType)) {
411+
case TLogGenerator::EType::Insert:
412+
case TLogGenerator::EType::Upsert:
413+
case TLogGenerator::EType::BulkUpsert:
414+
415+
416+
Cerr << "TimestampDevPassed: " << timestampDevPassed << "\n";
417+
Cerr << "DateFromPassed: " << dateFromPassed << "\n";
418+
Cerr << "DateToPassed: " << dateToPassed << "\n";
419+
420+
if (!timestampDevPassed && (!dateFromPassed || !dateToPassed)) {
421+
throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to";
422+
}
423+
424+
if (timestampDevPassed && (dateFromPassed || dateToPassed)) {
425+
throw yexception() << "The `timestamp_deviation` and `date-from`, `date-to` are mutually exclusive and shouldn't be provided at once";
426+
}
427+
428+
if ((dateFromPassed && !dateToPassed) || (!dateFromPassed && dateToPassed)) {
429+
throw yexception() << "The `date-from` and `date-to` parameters must be provided together to specify the interval for uniform PK generation";
430+
}
431+
432+
if (dateFromPassed && dateToPassed && *TimestampDateFrom >= *TimestampDateTo) {
433+
throw yexception() << "Invalid interval [`date-from`, `date-to`)";
434+
}
435+
436+
break;
437+
case TLogGenerator::EType::Select:
438+
439+
break;
440+
}
441+
break;
442+
case TWorkloadParams::ECommandType::Clean:
443+
break;
444+
case TWorkloadParams::ECommandType::Root:
445+
break;
446+
case TWorkloadParams::ECommandType::Import:
447+
break;
448+
}
449+
return;
450+
}
451+
363452
void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) {
364453
opts.AddLongOption('p', "path", "Path where benchmark tables are located")
365454
.Optional()
@@ -379,14 +468,7 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT
379468
.DefaultValue(PartitionSizeMb).StoreResult(&PartitionSizeMb);
380469
opts.AddLongOption("auto-partition", "Enable auto partitioning by load.")
381470
.DefaultValue(PartitionsByLoad).StoreResult(&PartitionsByLoad);
382-
opts.AddLongOption("len", "String len")
383-
.DefaultValue(StringLen).StoreResult(&StringLen);
384-
opts.AddLongOption("int-cols", "Number of int columns")
385-
.DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt);
386-
opts.AddLongOption("str-cols", "Number of string columns")
387-
.DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt);
388-
opts.AddLongOption("key-cols", "Number of key columns")
389-
.DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt);
471+
ConfigureOptsColumns(opts);
390472
opts.AddLongOption("ttl", "TTL for timestamp column in minutes")
391473
.DefaultValue(TimestampTtlMinutes).StoreResult(&TimestampTtlMinutes);
392474
opts.AddLongOption("store", "Storage type."
@@ -408,42 +490,14 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT
408490
case TLogGenerator::EType::Insert:
409491
case TLogGenerator::EType::Upsert:
410492
case TLogGenerator::EType::BulkUpsert:
411-
opts.AddLongOption("len", "String len")
412-
.DefaultValue(StringLen).StoreResult(&StringLen);
413-
opts.AddLongOption("int-cols", "Number of int columns")
414-
.DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt);
415-
opts.AddLongOption("str-cols", "Number of string columns")
416-
.DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt);
417-
opts.AddLongOption("key-cols", "Number of key columns")
418-
.DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt);
419-
opts.AddLongOption("rows", "Number of rows to upsert")
420-
.DefaultValue(RowsCnt).StoreResult(&RowsCnt);
421-
opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.")
422-
.DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes);
423-
opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted")
424-
.DefaultValue(0).StoreResult(&TimestampSubtract);
425-
opts.AddLongOption("null-percent", "Percent of nulls in generated data")
426-
.DefaultValue(NullPercent).StoreResult(&NullPercent);
493+
ConfigureOptsFillData(opts);
427494
break;
428495
case TLogGenerator::EType::Select:
429496
break;
430497
}
431498
break;
432499
case TWorkloadParams::ECommandType::Import:
433-
opts.AddLongOption("len", "String len")
434-
.DefaultValue(StringLen).StoreResult(&StringLen);
435-
opts.AddLongOption("int-cols", "Number of int columns")
436-
.DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt);
437-
opts.AddLongOption("str-cols", "Number of string columns")
438-
.DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt);
439-
opts.AddLongOption("key-cols", "Number of key columns")
440-
.DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt);
441-
opts.AddLongOption("rows", "Number of rows to upsert")
442-
.DefaultValue(RowsCnt).StoreResult(&RowsCnt);
443-
opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.")
444-
.DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes);
445-
opts.AddLongOption("null-percent", "Percent of nulls in generated data")
446-
.DefaultValue(NullPercent).StoreResult(&NullPercent);
500+
ConfigureOptsFillData(opts);
447501
break;
448502
default:
449503
break;

‎ydb/library/workload/log/log.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ class TLogWorkloadParams : public TWorkloadParams {
2424
ui64 IntColumnsCnt = 0;
2525
ui64 KeyColumnsCnt = 0;
2626
ui64 TimestampStandardDeviationMinutes = 0;
27+
TMaybe<ui64> TimestampDateFrom;
28+
TMaybe<ui64> TimestampDateTo;
2729
ui64 TimestampTtlMinutes = 0;
2830
ui64 TimestampSubtract = 0;
2931
ui64 RowsCnt = 1;
@@ -34,6 +36,11 @@ class TLogWorkloadParams : public TWorkloadParams {
3436

3537
YDB_READONLY(EStoreType, StoreType, EStoreType::Row);
3638
TWorkloadDataInitializer::TList CreateDataInitializers() const override;
39+
40+
void Validate(const ECommandType commandType, int workloadType) const override;
41+
private:
42+
void ConfigureOptsFillData(NLastGetopt::TOpts& opts);
43+
void ConfigureOptsColumns(NLastGetopt::TOpts& opts);
3744
};
3845

3946
class TLogGenerator final: public TWorkloadQueryGeneratorBase<TLogWorkloadParams> {

‎ydb/public/lib/ydb_cli/commands/ydb_workload.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@ int TWorkloadCommand::RunWorkload(NYdbWorkload::IWorkloadQueryGenerator& workloa
279279
StopTime = StartTime + TDuration::Seconds(TotalSec);
280280

281281
NPar::LocalExecutor().RunAdditionalThreads(Threads);
282+
282283
auto futures = NPar::LocalExecutor().ExecRangeWithFutures([this, &workloadGen, type](int id) {
283284
try {
284285
WorkerFn(id, workloadGen, type);
@@ -351,6 +352,7 @@ int TWorkloadCommandRun::Run(TConfig& config) {
351352
PrepareForRun(config);
352353
Params.DbPath = config.Database;
353354
auto workloadGen = Params.CreateGenerator();
355+
Params.Validate(NYdbWorkload::TWorkloadParams::ECommandType::Run, Type);
354356
return RunWorkload(*workloadGen, Type);
355357
}
356358

0 commit comments

Comments
 (0)