Skip to content

Commit 6a3722d

Browse files
EmgarikoEmgariko
authored
Merge 65cb9c8 into 805752b 
2 parents 805752b + 65cb9c8 commit 6a3722d

File tree

4 files changed

+80
-39
lines changed

4 files changed

+80
-39
lines changed

‎ydb/library/workload/abstract/workload_query_generator.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ class TWorkloadParams {
178178
}
179179
virtual TString GetWorkloadName() const = 0;
180180

181+
virtual void Validate() const {};
181182
public:
182183
ui64 BulkSize = 10000;
183184
std::string DbPath;

‎ydb/library/workload/log/log.cpp‎

Lines changed: 70 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
#include <library/cpp/resource/resource.h>
55
#include <util/datetime/base.h>
66
#include <util/generic/guid.h>
7+
#include <util/random/entropy.h>
8+
#include <util/random/mersenne.h>
79
#include <util/random/normal.h>
810
#include <util/random/random.h>
911
#include <util/string/split.h>
@@ -252,6 +254,11 @@ class TRandomLogGenerator {
252254
return result.str();
253255
}
254256

257+
TInstant UniformInstant(ui64 from, ui64 to) const {
258+
TMersenne<ui64> rnd(Seed());
259+
return TInstant::FromValue(rnd.Uniform(from, to));
260+
}
261+
255262
TInstant RandomInstant() const {
256263
auto result = TInstant::Now() - TDuration::Seconds(Params.TimestampSubtract);
257264
i64 millisecondsDiff = 60 * 1000 * NormalRandom<double>(0., Params.TimestampStandardDeviationMinutes);
@@ -279,7 +286,7 @@ class TRandomLogGenerator {
279286
for (size_t row = 0; row < count; ++row) {
280287
result.emplace_back();
281288
result.back().LogId = CreateGuidAsString().c_str();
282-
result.back().Ts = RandomInstant();
289+
result.back().Ts = !!Params.TimestampDateFrom && !!Params.TimestampDateTo ? UniformInstant(*Params.TimestampDateFrom, *Params.TimestampDateTo) : RandomInstant();
283290
result.back().Level = RandomNumber<ui32>(10);
284291
result.back().ServiceName = RandomWord(false);
285292
result.back().Component = RandomWord(true);
@@ -360,6 +367,65 @@ TQueryInfoList TLogGenerator::GetWorkload(int type) {
360367
}
361368
}
362369

370+
void TLogWorkloadParams::ConfigureOptsColumns(NLastGetopt::TOpts& opts) {
371+
opts.AddLongOption("len", "String len")
372+
.DefaultValue(StringLen).StoreResult(&StringLen);
373+
opts.AddLongOption("int-cols", "Number of int columns")
374+
.DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt);
375+
opts.AddLongOption("str-cols", "Number of string columns")
376+
.DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt);
377+
opts.AddLongOption("key-cols", "Number of key columns")
378+
.DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt);
379+
}
380+
381+
void TLogWorkloadParams::ConfigureOptsFillData(NLastGetopt::TOpts& opts) {
382+
ConfigureOptsColumns(opts);
383+
opts.AddLongOption("rows", "Number of rows to upsert")
384+
.DefaultValue(RowsCnt).StoreResult(&RowsCnt);
385+
opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.")
386+
.DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes);
387+
opts.AddLongOption("date-from", "Left boundary of the interval to generate "
388+
"timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' "
389+
"should be passed as well. This option is mutually exclusive with 'timestamp_deviation'")
390+
.StoreResult(&TimestampDateFrom);
391+
opts.AddLongOption("date-to", "Right boundary of the interval to generate "
392+
"timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' "
393+
"should be passed as well. This option is mutually exclusive with 'timestamp_deviation'")
394+
.StoreResult(&TimestampDateTo);
395+
opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted")
396+
.DefaultValue(0).StoreResult(&TimestampSubtract);
397+
opts.AddLongOption("null-percent", "Percent of nulls in generated data")
398+
.DefaultValue(NullPercent).StoreResult(&NullPercent);
399+
}
400+
401+
void TLogWorkloadParams::Validate() const {
402+
const bool timestampDevPassed = TimestampStandardDeviationMinutes;
403+
const bool dateFromPassed = !!TimestampDateFrom;
404+
const bool dateToPassed = !!TimestampDateTo;
405+
406+
Cerr << "TimestampDevPassed: " << timestampDevPassed << "\n";
407+
Cerr << "DateFromPassed: " << dateFromPassed << "\n";
408+
Cerr << "DateToPassed: " << dateToPassed << "\n";
409+
410+
if (!timestampDevPassed && (!dateFromPassed || !dateToPassed)) {
411+
throw yexception() << "One of parameter should be provided - timestamp_deviation or date-from and date-to";
412+
}
413+
414+
if (timestampDevPassed && (dateFromPassed || dateToPassed)) {
415+
throw yexception() << "The `timestamp_deviation` and `date-from`, `date-to` are mutually exclusive and shouldn't be provided at once";
416+
}
417+
418+
if ((dateFromPassed && !dateToPassed) || (!dateFromPassed && dateToPassed)) {
419+
throw yexception() << "The `date-from` and `date-to` parameters must be provided together to specify the interval for uniform PK generation";
420+
}
421+
422+
if (dateFromPassed && dateToPassed && *TimestampDateFrom >= *TimestampDateTo) {
423+
throw yexception() << "Invalid interval [`date-from`, `date-to`)";
424+
}
425+
426+
return;
427+
}
428+
363429
void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) {
364430
opts.AddLongOption('p', "path", "Path where benchmark tables are located")
365431
.Optional()
@@ -379,14 +445,7 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT
379445
.DefaultValue(PartitionSizeMb).StoreResult(&PartitionSizeMb);
380446
opts.AddLongOption("auto-partition", "Enable auto partitioning by load.")
381447
.DefaultValue(PartitionsByLoad).StoreResult(&PartitionsByLoad);
382-
opts.AddLongOption("len", "String len")
383-
.DefaultValue(StringLen).StoreResult(&StringLen);
384-
opts.AddLongOption("int-cols", "Number of int columns")
385-
.DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt);
386-
opts.AddLongOption("str-cols", "Number of string columns")
387-
.DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt);
388-
opts.AddLongOption("key-cols", "Number of key columns")
389-
.DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt);
448+
ConfigureOptsColumns(opts);
390449
opts.AddLongOption("ttl", "TTL for timestamp column in minutes")
391450
.DefaultValue(TimestampTtlMinutes).StoreResult(&TimestampTtlMinutes);
392451
opts.AddLongOption("store", "Storage type."
@@ -408,42 +467,14 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT
408467
case TLogGenerator::EType::Insert:
409468
case TLogGenerator::EType::Upsert:
410469
case TLogGenerator::EType::BulkUpsert:
411-
opts.AddLongOption("len", "String len")
412-
.DefaultValue(StringLen).StoreResult(&StringLen);
413-
opts.AddLongOption("int-cols", "Number of int columns")
414-
.DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt);
415-
opts.AddLongOption("str-cols", "Number of string columns")
416-
.DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt);
417-
opts.AddLongOption("key-cols", "Number of key columns")
418-
.DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt);
419-
opts.AddLongOption("rows", "Number of rows to upsert")
420-
.DefaultValue(RowsCnt).StoreResult(&RowsCnt);
421-
opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.")
422-
.DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes);
423-
opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted")
424-
.DefaultValue(0).StoreResult(&TimestampSubtract);
425-
opts.AddLongOption("null-percent", "Percent of nulls in generated data")
426-
.DefaultValue(NullPercent).StoreResult(&NullPercent);
470+
ConfigureOptsFillData(opts);
427471
break;
428472
case TLogGenerator::EType::Select:
429473
break;
430474
}
431475
break;
432476
case TWorkloadParams::ECommandType::Import:
433-
opts.AddLongOption("len", "String len")
434-
.DefaultValue(StringLen).StoreResult(&StringLen);
435-
opts.AddLongOption("int-cols", "Number of int columns")
436-
.DefaultValue(IntColumnsCnt).StoreResult(&IntColumnsCnt);
437-
opts.AddLongOption("str-cols", "Number of string columns")
438-
.DefaultValue(StrColumnsCnt).StoreResult(&StrColumnsCnt);
439-
opts.AddLongOption("key-cols", "Number of key columns")
440-
.DefaultValue(KeyColumnsCnt).StoreResult(&KeyColumnsCnt);
441-
opts.AddLongOption("rows", "Number of rows to upsert")
442-
.DefaultValue(RowsCnt).StoreResult(&RowsCnt);
443-
opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.")
444-
.DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes);
445-
opts.AddLongOption("null-percent", "Percent of nulls in generated data")
446-
.DefaultValue(NullPercent).StoreResult(&NullPercent);
477+
ConfigureOptsFillData(opts);
447478
break;
448479
default:
449480
break;

‎ydb/library/workload/log/log.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ class TLogWorkloadParams : public TWorkloadParams {
2424
ui64 IntColumnsCnt = 0;
2525
ui64 KeyColumnsCnt = 0;
2626
ui64 TimestampStandardDeviationMinutes = 0;
27+
TMaybe<ui64> TimestampDateFrom;
28+
TMaybe<ui64> TimestampDateTo;
2729
ui64 TimestampTtlMinutes = 0;
2830
ui64 TimestampSubtract = 0;
2931
ui64 RowsCnt = 1;
@@ -34,6 +36,11 @@ class TLogWorkloadParams : public TWorkloadParams {
3436

3537
YDB_READONLY(EStoreType, StoreType, EStoreType::Row);
3638
TWorkloadDataInitializer::TList CreateDataInitializers() const override;
39+
40+
void Validate() const override;
41+
private:
42+
void ConfigureOptsFillData(NLastGetopt::TOpts& opts);
43+
void ConfigureOptsColumns(NLastGetopt::TOpts& opts);
3744
};
3845

3946
class TLogGenerator final: public TWorkloadQueryGeneratorBase<TLogWorkloadParams> {

‎ydb/public/lib/ydb_cli/commands/ydb_workload.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@ int TWorkloadCommand::RunWorkload(NYdbWorkload::IWorkloadQueryGenerator& workloa
279279
StopTime = StartTime + TDuration::Seconds(TotalSec);
280280

281281
NPar::LocalExecutor().RunAdditionalThreads(Threads);
282+
282283
auto futures = NPar::LocalExecutor().ExecRangeWithFutures([this, &workloadGen, type](int id) {
283284
try {
284285
WorkerFn(id, workloadGen, type);
@@ -351,6 +352,7 @@ int TWorkloadCommandRun::Run(TConfig& config) {
351352
PrepareForRun(config);
352353
Params.DbPath = config.Database;
353354
auto workloadGen = Params.CreateGenerator();
355+
Params.Validate();
354356
return RunWorkload(*workloadGen, Type);
355357
}
356358

0 commit comments

Comments
 (0)