Skip to content

Commit 6bd8708

Browse files
iddqdexiddqdex
authored
Small speedup tpcds generator (#10424)
1 parent d14474e commit 6bd8708

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+109
-68
lines changed

‎ydb/library/workload/tpcds/data_generator.cpp‎

Lines changed: 24 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -135,35 +135,34 @@ TTpcdsWorkloadDataInitializerGenerator::TBulkDataGenerator::TDataPortions TTpcds
135135
ctxs.emplace_back(*this, tdef->nParam);
136136
}
137137

138-
with_lock(Lock) {
139-
ds_key_t firstRow;
140-
ds_key_t rowCount;
141-
split_work(TableNum, &firstRow, &rowCount);
142-
if (!Generated) {
143-
ui32 toSkip = firstRow - 1;
144-
if (!!Owner.StateProcessor && Owner.StateProcessor->GetState().contains(GetName())) {
145-
Generated = Owner.StateProcessor->GetState().at(TString(GetName())).Position;
146-
toSkip += Generated;
147-
}
148-
if (toSkip) {
149-
row_skip(TableNum, toSkip);
150-
if (tdef->flags & FL_PARENT) {
151-
row_skip(tdef->nParam, toSkip);
152-
}
153-
}
154-
if (tdef->flags & FL_SMALL) {
155-
resetCountCount();
138+
auto g = Guard(Lock);
139+
ds_key_t firstRow;
140+
ds_key_t rowCount;
141+
split_work(TableNum, &firstRow, &rowCount);
142+
if (!Generated) {
143+
ui32 toSkip = firstRow - 1;
144+
if (!!Owner.StateProcessor && Owner.StateProcessor->GetState().contains(GetName())) {
145+
Generated = Owner.StateProcessor->GetState().at(TString(GetName())).Position;
146+
toSkip += Generated;
147+
}
148+
if (toSkip) {
149+
row_skip(TableNum, toSkip);
150+
if (tdef->flags & FL_PARENT) {
151+
row_skip(tdef->nParam, toSkip);
156152
}
157153
}
158-
const auto count = TableSize > Generated ? std::min(ui64(TableSize - Generated), Owner.Params.BulkSize) : 0;
159-
if (!count) {
160-
return result;
154+
if (tdef->flags & FL_SMALL) {
155+
resetCountCount();
161156
}
162-
ctxs.front().SetCount(count);
163-
ctxs.front().SetStart(firstRow + Generated);
164-
Generated += count;
165-
GenerateRows(ctxs);
166157
}
158+
const auto count = TableSize > Generated ? std::min(ui64(TableSize - Generated), Owner.Params.BulkSize) : 0;
159+
if (!count) {
160+
return result;
161+
}
162+
ctxs.front().SetCount(count);
163+
ctxs.front().SetStart(firstRow + Generated);
164+
Generated += count;
165+
GenerateRows(ctxs, std::move(g));
167166
for(auto& ctx: ctxs) {
168167
ctx.AppendPortions(result);
169168
}

‎ydb/library/workload/tpcds/data_generator.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ class TTpcdsWorkloadDataInitializerGenerator: public TWorkloadDataInitializerBas
5050

5151
using TContexts = TVector<TContext>;
5252

53-
virtual void GenerateRows(TContexts& ctxs) = 0;
53+
virtual void GenerateRows(TContexts& ctxs, TGuard<TAdaptiveLock>&& g) = 0;
5454

5555
int TableNum;
5656
ui64 Generated = 0;

‎ydb/library/workload/tpcds/dg_call_center.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,15 @@ TTpcDSGeneratorCallCenter::TTpcDSGeneratorCallCenter(const TTpcdsWorkloadDataIni
1313
: TBulkDataGenerator(owner, CALL_CENTER)
1414
{}
1515

16-
void TTpcDSGeneratorCallCenter::GenerateRows(TContexts& ctxs) {
16+
void TTpcDSGeneratorCallCenter::GenerateRows(TContexts& ctxs, TGuard<TAdaptiveLock>&& g) {
1717
TVector<CALL_CENTER_TBL> callCenterList(ctxs.front().GetCount());
1818
for (ui64 i = 0; i < ctxs.front().GetCount(); ++i) {
1919
mk_w_call_center(NULL, ctxs.front().GetStart() + i);
2020
callCenterList[i] = g_w_call_center;
2121
tpcds_row_stop(TableNum);
2222
}
23+
g.Release();
24+
2325
TCsvItemWriter<CALL_CENTER_TBL> writer(ctxs.front().GetCsv().Out);
2426
CSV_WRITER_REGISTER_SIMPLE_FIELD_KEY(writer, cc_call_center_sk);
2527
CSV_WRITER_REGISTER_SIMPLE_FIELD_STRING(writer, cc_call_center_id);

‎ydb/library/workload/tpcds/dg_call_center.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ class TTpcDSGeneratorCallCenter : public TTpcdsWorkloadDataInitializerGenerator:
77
explicit TTpcDSGeneratorCallCenter(const TTpcdsWorkloadDataInitializerGenerator& owner);
88

99
protected:
10-
virtual void GenerateRows(TContexts& ctxs) override;
10+
virtual void GenerateRows(TContexts& ctxs, TGuard<TAdaptiveLock>&& g) override;
1111
static const TFactory::TRegistrator<TTpcDSGeneratorCallCenter> Registrar;
1212
};
1313

‎ydb/library/workload/tpcds/dg_catalog_page.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,15 @@ TTpcDSGeneratorCatalogPage::TTpcDSGeneratorCatalogPage(const TTpcdsWorkloadDataI
1313
: TBulkDataGenerator(owner, CATALOG_PAGE)
1414
{}
1515

16-
void TTpcDSGeneratorCatalogPage::GenerateRows(TContexts& ctxs) {
16+
void TTpcDSGeneratorCatalogPage::GenerateRows(TContexts& ctxs, TGuard<TAdaptiveLock>&& g) {
1717
TVector<CATALOG_PAGE_TBL> catalogPageList(ctxs.front().GetCount());
1818
for (ui64 i = 0; i < ctxs.front().GetCount(); ++i) {
1919
mk_w_catalog_page(NULL, ctxs.front().GetStart() + i);
2020
catalogPageList[i] = g_w_catalog_page;
2121
tpcds_row_stop(TableNum);
2222
}
23+
g.Release();
24+
2325
TCsvItemWriter<CATALOG_PAGE_TBL> writer(ctxs.front().GetCsv().Out);
2426
CSV_WRITER_REGISTER_SIMPLE_FIELD_KEY(writer, cp_catalog_page_sk);
2527
CSV_WRITER_REGISTER_SIMPLE_FIELD_STRING(writer, cp_catalog_page_id);

‎ydb/library/workload/tpcds/dg_catalog_page.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ class TTpcDSGeneratorCatalogPage : public TTpcdsWorkloadDataInitializerGenerator
77
explicit TTpcDSGeneratorCatalogPage(const TTpcdsWorkloadDataInitializerGenerator& owner);
88

99
protected:
10-
virtual void GenerateRows(TContexts& ctxs) override;
10+
virtual void GenerateRows(TContexts& ctxs, TGuard<TAdaptiveLock>&& g) override;
1111
static const TFactory::TRegistrator<TTpcDSGeneratorCatalogPage> Registrar;
1212
};
1313

‎ydb/library/workload/tpcds/dg_catalog_sales.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ TTpcDSGeneratorCatalogSales::TTpcDSGeneratorCatalogSales(const TTpcdsWorkloadDat
100100
: TBulkDataGenerator(owner, CATALOG_SALES)
101101
{}
102102

103-
void TTpcDSGeneratorCatalogSales::GenerateRows(TContexts& ctxs) {
103+
void TTpcDSGeneratorCatalogSales::GenerateRows(TContexts& ctxs, TGuard<TAdaptiveLock>&& g) {
104104
TVector<W_CATALOG_SALES_TBL> catalogSalesList;
105105
TVector<W_CATALOG_RETURNS_TBL> catalogReturnsList;
106106
catalogReturnsList.reserve(ctxs.front().GetCount() * 14);
@@ -115,6 +115,8 @@ void TTpcDSGeneratorCatalogSales::GenerateRows(TContexts& ctxs) {
115115
}
116116
tpcds_row_stop(TableNum);
117117
}
118+
g.Release();
119+
118120
TCsvItemWriter<W_CATALOG_SALES_TBL> writerSales(ctxs.front().GetCsv().Out);
119121
CSV_WRITER_REGISTER_FIELD(writerSales, "cs_item_sk", cs_sold_item_sk);
120122
CSV_WRITER_REGISTER_SIMPLE_FIELD_KEY(writerSales, cs_order_number);

‎ydb/library/workload/tpcds/dg_catalog_sales.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ class TTpcDSGeneratorCatalogSales : public TTpcdsWorkloadDataInitializerGenerato
77
explicit TTpcDSGeneratorCatalogSales(const TTpcdsWorkloadDataInitializerGenerator& owner);
88

99
protected:
10-
virtual void GenerateRows(TContexts& ctxs) override;
10+
virtual void GenerateRows(TContexts& ctxs, TGuard<TAdaptiveLock>&& g) override;
1111
static const TFactory::TRegistrator<TTpcDSGeneratorCatalogSales> Registrar;
1212
};
1313

‎ydb/library/workload/tpcds/dg_customer.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,14 @@ TTpcDSGeneratorCustomer::TTpcDSGeneratorCustomer(const TTpcdsWorkloadDataInitial
1212
: TBulkDataGenerator(owner, CUSTOMER)
1313
{}
1414

15-
void TTpcDSGeneratorCustomer::GenerateRows(TContexts& ctxs) {
15+
void TTpcDSGeneratorCustomer::GenerateRows(TContexts& ctxs, TGuard<TAdaptiveLock>&& g) {
1616
TVector<W_CUSTOMER_TBL> customerList(ctxs.front().GetCount());
1717
for (ui64 i = 0; i < ctxs.front().GetCount(); ++i) {
1818
mk_w_customer(&customerList[i], ctxs.front().GetStart() + i);
1919
tpcds_row_stop(TableNum);
2020
}
21+
g.Release();
22+
2123
TCsvItemWriter<W_CUSTOMER_TBL> writer(ctxs.front().GetCsv().Out);
2224
CSV_WRITER_REGISTER_SIMPLE_FIELD_KEY(writer, c_customer_sk);
2325
CSV_WRITER_REGISTER_SIMPLE_FIELD_STRING(writer, c_customer_id);

‎ydb/library/workload/tpcds/dg_customer.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ class TTpcDSGeneratorCustomer : public TTpcdsWorkloadDataInitializerGenerator::T
77
explicit TTpcDSGeneratorCustomer(const TTpcdsWorkloadDataInitializerGenerator& owner);
88

99
protected:
10-
virtual void GenerateRows(TContexts& ctxs) override;
10+
virtual void GenerateRows(TContexts& ctxs, TGuard<TAdaptiveLock>&& g) override;
1111
static const TFactory::TRegistrator<TTpcDSGeneratorCustomer> Registrar;
1212
};
1313

0 commit comments

Comments
 (0)