Merge 39fefbf into 6e24151

Author: Emgariko

ydb/library/workload/abstract/workload_query_generator.h

@@ -6,6 +6,7 @@
 #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/value/value.h>
 #include <ydb/library/accessor/accessor.h>
 #include <library/cpp/getopt/last_getopt.h>
+#include <ydb/public/lib/ydb_cli/common/command.h>
 
 #include <list>
 #include <string>
@@ -172,6 +173,7 @@ class TWorkloadParams {
     virtual ~TWorkloadParams() = default;
     virtual void ConfigureOpts(NLastGetopt::TOpts& /*opts*/, const ECommandType /*commandType*/, int /*workloadType*/) {
     };
+    virtual void Parse(NYdb::NConsoleClient::TClientCommand::TConfig& /*config*/) {};
     virtual THolder<IWorkloadQueryGenerator> CreateGenerator() const = 0;
     virtual TWorkloadDataInitializer::TList CreateDataInitializers() const {
         return {};

ydb/library/workload/log/log.cpp

@@ -1,4 +1,6 @@
 #include "log.h"
+#include <util/random/entropy.h>
+#include <util/random/mersenne.h>
 #include <ydb/public/api/protos/ydb_formats.pb.h>
 #include <library/cpp/json/json_value.h>
 #include <library/cpp/resource/resource.h>
@@ -252,6 +254,11 @@ class TRandomLogGenerator {
         return result.str();
     }
 
+    TInstant UniformInstant(ui64 from, ui64 to) const {
+        TMersenne<ui64> rnd(Seed());
+        return TInstant::FromValue(rnd.Uniform(from, to));
+    }
+
     TInstant RandomInstant() const {
         auto result = TInstant::Now() - TDuration::Seconds(Params.TimestampSubtract);
         i64 millisecondsDiff = 60 * 1000 * NormalRandom<double>(0., Params.TimestampStandardDeviationMinutes);
@@ -279,7 +286,7 @@ class TRandomLogGenerator {
         for (size_t row = 0; row < count; ++row) {
             result.emplace_back();
             result.back().LogId = CreateGuidAsString().c_str();
-            result.back().Ts = RandomInstant();
+            result.back().Ts = Params.TimestampDateFrom.has_value() ? UniformInstant(*Params.TimestampDateFrom, *Params.TimestampDateTo) : RandomInstant();
             result.back().Level = RandomNumber<ui32>(10);
             result.back().ServiceName = RandomWord(false);
             result.back().Component = RandomWord(true);
@@ -419,7 +426,23 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT
             opts.AddLongOption("rows", "Number of rows to upsert")
                 .DefaultValue(RowsCnt).StoreResult(&RowsCnt);
             opts.AddLongOption("timestamp_deviation", "Standard deviation. For each timestamp, a random variable with a specified standard deviation in minutes is added.")
-                .DefaultValue(TimestampStandardDeviationMinutes).StoreResult(&TimestampStandardDeviationMinutes);
+                // .DefaultValue(TimestampStandardDeviationMinutes)
+                .Optional()
+                .StoreResult(&TimestampStandardDeviationMinutes);
+            // TODO: maybe it shoudn't be optional
+            
+            opts.AddLongOption("date-from", "Left boundary of the interval to generate "
+                "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-to' "
+                "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'")
+                .Optional().StoreResult(&TimestampDateFrom);
+            opts.AddLongOption("date-to", "Right boundary of the interval to generate "
+                "timestamp uniformly from specified interval. Presents as seconds since epoch. Once this option passed, 'date-from' "
+                "should be passed as well. This option is mutually exclusive with 'timestamp_deviation'")
+                .Optional().StoreResult(&TimestampDateTo);
+
+            // opts.MutuallyExclusive("timestamp_deviation", "date-from");
+            // opts.MutuallyExclusive("timestamp_deviation", "date-to");
+    
             opts.AddLongOption("timestamp_subtract", "Value in seconds to subtract from timestamp. For each timestamp, this value in seconds is subtracted")
                 .DefaultValue(0).StoreResult(&TimestampSubtract);
             opts.AddLongOption("null-percent", "Percent of nulls in generated data")
@@ -450,6 +473,21 @@ void TLogWorkloadParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandT
     }
 }
 
+void TLogWorkloadParams::Parse(NYdb::NConsoleClient::TClientCommand::TConfig& config) {
+    // TODO: should I check if it's a specific command type: run ?
+    auto timestamp_dev_passed = config.ParseResult->Has("timestamp_deviation");
+    auto date_from_passed = config.ParseResult->Has("date-from");
+    auto date_to_passed = config.ParseResult->Has("date-to");
+
+    if (timestamp_dev_passed && (date_from_passed || date_to_passed)) {
+        throw yexception() << "The `timestamp_deviation` and `date_from`, `date_to` are mutually exclusive and shouldn't be passed at once";
+    }
+
+    if ((date_from_passed && !date_to_passed) || (!date_from_passed && date_to_passed)) {
+        throw yexception() << "The `date_from` and `date_to` parameters must be provided together to specify the interval for uniform PK generation";
+    }
+}
+
 THolder<IWorkloadQueryGenerator> TLogWorkloadParams::CreateGenerator() const {
     return MakeHolder<TLogGenerator>(this);
 }

ydb/library/workload/log/log.h

@@ -14,6 +14,7 @@ class TLogWorkloadParams : public TWorkloadParams {
     };
 
     void ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) override;
+    void Parse(NYdb::NConsoleClient::TClientCommand::TConfig& config) override;
     THolder<IWorkloadQueryGenerator> CreateGenerator() const override;
     TString GetWorkloadName() const override;
     ui64 MinPartitions = 40;
@@ -24,6 +25,8 @@ class TLogWorkloadParams : public TWorkloadParams {
     ui64 IntColumnsCnt = 0;
     ui64 KeyColumnsCnt = 0;
     ui64 TimestampStandardDeviationMinutes = 0;
+    std::optional<ui64> TimestampDateFrom{std::nullopt};
+    std::optional<ui64> TimestampDateTo{std::nullopt};
     ui64 TimestampTtlMinutes = 0;
     ui64 TimestampSubtract = 0;
     ui64 RowsCnt = 1;

ydb/library/workload/tpch/driver.h

@@ -6,7 +6,13 @@ extern "C" {
 
 #include <ydb/library/benchmarks/gen/tpch-dbgen/config.h>
 #include <ydb/library/benchmarks/gen/tpch-dbgen/release.h>
+#ifdef DT_CHR
+#undef DT_CHR
+#endif
 #include <ydb/library/benchmarks/gen/tpch-dbgen/dss.h>
+#ifdef DT_CHR
+#undef DT_CHR
+#endif
 #include <ydb/library/benchmarks/gen/tpch-dbgen/dsstypes.h>
 
 void InitTpchGen(DSS_HUGE scale);

ydb/public/lib/ydb_cli/commands/ydb_workload.cpp

@@ -374,6 +374,10 @@ void TWorkloadCommandBase::Config(TConfig& config) {
     Params.ConfigureOpts(config.Opts->GetOpts(), CommandType, Type);
 }
 
+void TWorkloadCommandBase::Parse(TConfig& config) {
+    Params.Parse(config);
+}
+
 int TWorkloadCommandBase::Run(TConfig& config) {
     if (!DryRun) {
         Driver = MakeHolder<NYdb::TDriver>(CreateDriver(config));

ydb/public/lib/ydb_cli/commands/ydb_workload.h

@@ -89,6 +89,7 @@ class TWorkloadCommandBase: public TYdbCommand {
         const TString& description = TString(),
         int type = 0);
     virtual void Config(TConfig& config) override;
+    virtual void Parse(TConfig& config) override;
     virtual int Run(TConfig& config) override final;
 
 protected: