Skip to content

Commit 1511453

Browse files
GazizonokiGazizonoki
committed
Fix
1 parent 50f738f commit 1511453

File tree

5 files changed

+69
-73
lines changed

5 files changed

+69
-73
lines changed

‎ydb/public/lib/ydb_cli/common/csv_parser.cpp‎

Lines changed: 55 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -8,33 +8,30 @@ namespace NYdb {
88
namespace NConsoleClient {
99
namespace {
1010

11+
struct TExceptionData {
12+
ui64 Line;
13+
std::optional<TString> ColumnName;
14+
};
15+
1116
class TCsvParseException : public TMisuseException {
1217
public:
13-
TCsvParseException() {
14-
if (ColumnName.has_value()) {
15-
*this << "Csv parsing error on line " << Line << " in column \"" << ColumnName << "\": ";
16-
} else {
17-
*this << "Csv parsing error on line " << Line << ": ";
18+
TCsvParseException(const TExceptionData& state) {
19+
*this << "Csv parsing error";
20+
if (state.Line != 0) {
21+
*this << " on line " << state.Line;
1822
}
23+
if (state.ColumnName.has_value()) {
24+
*this << " in column \"" << *state.ColumnName << "\"";
25+
}
26+
*this << ": ";
1927
}
20-
21-
static void NextLine() {
22-
++Line;
23-
}
24-
25-
static void SetColumnName(const std::optional<TString>& columnName) {
26-
ColumnName = columnName;
27-
}
28-
29-
private:
30-
inline static size_t Line = 1;
31-
inline static std::optional<TString> ColumnName;
3228
};
3329

3430
class TCsvToYdbConverter {
3531
public:
36-
explicit TCsvToYdbConverter(TTypeParser& parser, const std::optional<TString>& nullValue)
37-
: Parser(parser)
32+
explicit TCsvToYdbConverter(const TExceptionData& state, TTypeParser& parser, const std::optional<TString>& nullValue)
33+
: State(state)
34+
, Parser(parser)
3835
, NullValue(nullValue)
3936
{
4037
}
@@ -69,7 +66,7 @@ class TCsvToYdbConverter {
6966
}
7067
return static_cast<T>(value);
7168
} catch (std::exception& e) {
72-
throw TCsvParseException() << "Expected " << Parser.GetPrimitive() << " value, recieved: \"" << token << "\".";
69+
throw TCsvParseException(State) << "Expected " << Parser.GetPrimitive() << " value, recieved: \"" << token << "\".";
7370
}
7471
}
7572

@@ -166,7 +163,7 @@ class TCsvToYdbConverter {
166163
Builder.TzTimestamp(token);
167164
break;
168165
default:
169-
TCsvParseException() << "Unsupported primitive type: " << Parser.GetPrimitive();
166+
throw TCsvParseException(State) << "Unsupported primitive type: " << Parser.GetPrimitive();
170167
}
171168
}
172169

@@ -217,7 +214,7 @@ class TCsvToYdbConverter {
217214
break;
218215
}
219216
default:
220-
throw TCsvParseException() << "Unsupported type kind: " << Parser.GetKind();
217+
throw TCsvParseException(State) << "Unsupported type kind: " << Parser.GetKind();
221218
}
222219
}
223220

@@ -252,7 +249,7 @@ class TCsvToYdbConverter {
252249
break;
253250

254251
default:
255-
throw TCsvParseException() << "Unsupported type kind: " << Parser.GetKind();
252+
throw TCsvParseException(State) << "Unsupported type kind: " << Parser.GetKind();
256253
}
257254
}
258255

@@ -269,15 +266,15 @@ class TCsvToYdbConverter {
269266
if (token == "false") {
270267
return false;
271268
}
272-
throw TCsvParseException() << "Expected bool value: \"true\" or \"false\", recieved: \"" << token << "\".";
269+
throw TCsvParseException(State) << "Expected bool value: \"true\" or \"false\", recieved: \"" << token << "\".";
273270
}
274271

275272
void EnsureNull(TStringBuf token) const {
276273
if (!NullValue) {
277-
throw TCsvParseException() << "Expected null value instead of \"" << token << "\", but null value is not set.";
274+
throw TCsvParseException(State) << "Expected null value instead of \"" << token << "\", but null value is not set.";
278275
}
279276
if (token != NullValue) {
280-
throw TCsvParseException() << "Expected null value: \"" << NullValue << "\", recieved: \"" << token << "\".";
277+
throw TCsvParseException(State) << "Expected null value: \"" << NullValue << "\", recieved: \"" << token << "\".";
281278
}
282279
}
283280

@@ -287,11 +284,25 @@ class TCsvToYdbConverter {
287284
}
288285

289286
private:
287+
const TExceptionData State;
290288
TTypeParser& Parser;
291289
const std::optional<TString> NullValue = "";
292290
TValueBuilder Builder;
293291
};
294292

293+
TStringBuf Consume( const TExceptionData& state, NCsvFormat::CsvSplitter& splitter) {
294+
try {
295+
return splitter.Consume();
296+
} catch (std::exception& e) {
297+
throw TCsvParseException(state) << e.what();
298+
}
299+
}
300+
301+
TValue FieldToValue(const TExceptionData& state, TTypeParser& parser, TStringBuf token, const std::optional<TString>& nullValue) {
302+
TCsvToYdbConverter converter(state, parser, nullValue);
303+
return converter.Convert(token);
304+
}
305+
295306
}
296307

297308
TCsvParser::TCsvParser(TString&& headerRow, const char delimeter, const std::optional<TString>& nullValue,
@@ -318,26 +329,14 @@ TCsvParser::TCsvParser(TVector<TString>&& header, const char delimeter, const st
318329
{
319330
}
320331

321-
TValue TCsvParser::FieldToValue(TTypeParser& parser, TStringBuf token) const {
322-
TCsvToYdbConverter converter(parser, NullValue);
323-
return converter.Convert(token);
324-
}
325-
326-
TStringBuf TCsvParser::Consume(NCsvFormat::CsvSplitter& splitter) const {
327-
try {
328-
return splitter.Consume();
329-
} catch (std::exception& e) {
330-
throw TCsvParseException() << e.what();
331-
}
332-
}
333-
334-
void TCsvParser::GetParams(TString&& data, TParamsBuilder& builder) const {
332+
void TCsvParser::GetParams(ui64 line, TString&& data, TParamsBuilder& builder) const {
335333
NCsvFormat::CsvSplitter splitter(data, Delimeter);
334+
TExceptionData state{line, std::nullopt};
336335
auto headerIt = Header.begin();
337336
do {
338-
TStringBuf token = Consume(splitter);
337+
TStringBuf token = Consume(state, splitter);
339338
if (headerIt == Header.end()) {
340-
throw TCsvParseException() << "Header contains less fields than data. Header: \"" << HeaderRow << "\", data: \"" << data << "\"";
339+
throw TCsvParseException(state) << "Header contains less fields than data. Header: \"" << HeaderRow << "\", data: \"" << data << "\"";
341340
}
342341
TString fullname = "$" + *headerIt;
343342
auto paramIt = ParamTypes->find(fullname);
@@ -348,38 +347,37 @@ void TCsvParser::GetParams(TString&& data, TParamsBuilder& builder) const {
348347
if (ParamSources) {
349348
auto paramSource = ParamSources->find(fullname);
350349
if (paramSource != ParamSources->end()) {
351-
throw TCsvParseException() << "Parameter " << fullname << " value found in more than one source: stdin, " << paramSource->second << ".";
350+
throw TCsvParseException(state) << "Parameter " << fullname << " value found in more than one source: stdin, " << paramSource->second << ".";
352351
}
353352
}
354353
TTypeParser parser(paramIt->second);
355-
TCsvParseException::SetColumnName(*headerIt);
356-
builder.AddParam(fullname, FieldToValue(parser, token));
357-
TCsvParseException::SetColumnName(std::nullopt);
354+
state.ColumnName = *headerIt;
355+
builder.AddParam(fullname, FieldToValue(state, parser, token, NullValue));
356+
state.ColumnName = std::nullopt;
358357
++headerIt;
359358
} while (splitter.Step());
360359

361360
if (headerIt != Header.end()) {
362-
throw TCsvParseException() << "Header contains more fields than data. Header: \"" << HeaderRow << "\", data: \"" << data << "\"";
361+
throw TCsvParseException(state) << "Header contains more fields than data. Header: \"" << HeaderRow << "\", data: \"" << data << "\"";
363362
}
364-
365-
TCsvParseException::NextLine();
366363
}
367364

368-
void TCsvParser::GetValue(TString&& data, TValueBuilder& builder, const TType& type) const {
365+
void TCsvParser::GetValue(ui64 line, TString&& data, TValueBuilder& builder, const TType& type) const {
369366
NCsvFormat::CsvSplitter splitter(data, Delimeter);
367+
TExceptionData state{line, std::nullopt};
370368
auto headerIt = Header.cbegin();
371369
std::map<TString, TStringBuf> fields;
372370
do {
373-
TStringBuf token = Consume(splitter);;
371+
TStringBuf token = Consume(state, splitter);
374372
if (headerIt == Header.cend()) {
375-
throw TCsvParseException() << "Header contains less fields than data. Header: \"" << HeaderRow << "\", data: \"" << data << "\"";
373+
throw TCsvParseException(state) << "Header contains less fields than data. Header: \"" << HeaderRow << "\", data: \"" << data << "\"";
376374
}
377375
fields[*headerIt] = token;
378376
++headerIt;
379377
} while (splitter.Step());
380378

381379
if (headerIt != Header.cend()) {
382-
throw TCsvParseException() << "Header contains more fields than data. Header: \"" << HeaderRow << "\", data: \"" << data << "\"";
380+
throw TCsvParseException(state) << "Header contains more fields than data. Header: \"" << HeaderRow << "\", data: \"" << data << "\"";
383381
}
384382

385383
builder.BeginStruct();
@@ -392,17 +390,15 @@ void TCsvParser::GetValue(TString&& data, TValueBuilder& builder, const TType& t
392390
}
393391
auto fieldIt = fields.find(name);
394392
if (fieldIt == fields.end()) {
395-
throw TCsvParseException() << "No member \"" << name << "\" in csv string for YDB struct type";
393+
throw TCsvParseException(state) << "No member \"" << name << "\" in csv string for YDB struct type";
396394
}
397-
TCsvParseException::SetColumnName(name);
398-
builder.AddMember(name, FieldToValue(parser, fieldIt->second));
399-
TCsvParseException::SetColumnName(std::nullopt);
395+
state.ColumnName = name;
396+
builder.AddMember(name, FieldToValue(state, parser, fieldIt->second, NullValue));
397+
state.ColumnName = std::nullopt;
400398
}
401399

402400
parser.CloseStruct();
403401
builder.EndStruct();
404-
405-
TCsvParseException::NextLine();
406402
}
407403

408404
TType TCsvParser::GetColumnsType() const {

‎ydb/public/lib/ydb_cli/common/csv_parser.h‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,11 @@ class TCsvParser {
2424
const std::map<TString, TType>* paramTypes = nullptr,
2525
const std::map<TString, TString>* paramSources = nullptr);
2626

27-
void GetParams(TString&& data, TParamsBuilder& builder) const;
28-
void GetValue(TString&& data, TValueBuilder& builder, const TType& type) const;
27+
void GetParams(ui64 line, TString&& data, TParamsBuilder& builder) const;
28+
void GetValue(ui64 line, TString&& data, TValueBuilder& builder, const TType& type) const;
2929
TType GetColumnsType() const;
30-
TStringBuf Consume(NCsvFormat::CsvSplitter& splitter) const;
3130

3231
private:
33-
TValue FieldToValue(TTypeParser& parser, TStringBuf token) const;
34-
3532
TVector<TString> Header;
3633
TString HeaderRow;
3734
char Delimeter;

‎ydb/public/lib/ydb_cli/common/parameters.cpp‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ void TCommandWithParameters::AddParams(TParamsBuilder& paramBuilder) {
204204
bool TCommandWithParameters::GetNextParams(THolder<TParamsBuilder>& paramBuilder) {
205205
paramBuilder = MakeHolder<TParamsBuilder>();
206206
if (IsFirstEncounter) {
207+
Row = SkipRows;
207208
IsFirstEncounter = false;
208209
ParamTypes = ValidateResult->GetParameterTypes();
209210
if (IsStdinInteractive()) {
@@ -235,6 +236,7 @@ bool TCommandWithParameters::GetNextParams(THolder<TParamsBuilder>& paramBuilder
235236
if (IsStdinInteractive()) {
236237
return false;
237238
}
239+
++Row;
238240

239241
AddParams(*paramBuilder);
240242
if (BatchMode == EBatchMode::Iterative) {
@@ -257,7 +259,7 @@ bool TCommandWithParameters::GetNextParams(THolder<TParamsBuilder>& paramBuilder
257259
}
258260
case EOutputFormat::Csv:
259261
case EOutputFormat::Tsv: {
260-
CsvParser.GetParams(std::move(*data), *paramBuilder);
262+
CsvParser.GetParams(Row, std::move(*data), *paramBuilder);
261263
break;
262264
}
263265
default:
@@ -300,7 +302,7 @@ bool TCommandWithParameters::GetNextParams(THolder<TParamsBuilder>& paramBuilder
300302
case EOutputFormat::Csv:
301303
case EOutputFormat::Tsv: {
302304
TValueBuilder valueBuilder;
303-
CsvParser.GetValue(std::move(*data), valueBuilder, type);
305+
CsvParser.GetValue(Row, std::move(*data), valueBuilder, type);
304306
paramBuilder->AddParam(fullname, valueBuilder.Build());
305307
break;
306308
}
@@ -379,7 +381,7 @@ bool TCommandWithParameters::GetNextParams(THolder<TParamsBuilder>& paramBuilder
379381
case EOutputFormat::Csv:
380382
case EOutputFormat::Tsv: {
381383
valueBuilder.AddListItem();
382-
CsvParser.GetValue(std::move(*data), valueBuilder, type.GetProto().list_type().item());
384+
CsvParser.GetValue(Row, std::move(*data), valueBuilder, type.GetProto().list_type().item());
383385
break;
384386
}
385387
default:

‎ydb/public/lib/ydb_cli/common/parameters.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ class TCommandWithParameters : public TCommandWithExamples, public TCommandWithF
4141
THolder<IParamStream> Input;
4242
bool IsFirstEncounter = true;
4343
size_t SkipRows = 0;
44+
ui64 Row = 0;
4445
char Delimiter;
4546
TCsvParser CsvParser;
4647

‎ydb/public/lib/ydb_cli/import/import.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -431,12 +431,12 @@ TStatus TImportFileClient::UpsertCsv(IInputStream& input, const TString& dbPath,
431431
std::vector<TAsyncStatus> inFlightRequests;
432432
std::vector<TString> buffer;
433433

434-
auto upsertCsv = [&](std::vector<TString>&& buffer) {
434+
auto upsertCsv = [&](ui64 row, std::vector<TString>&& buffer) {
435435
TValueBuilder builder;
436436
builder.BeginList();
437437
for (auto&& line : buffer) {
438438
builder.AddListItem();
439-
parser.GetValue(std::move(line), builder, lineType);
439+
parser.GetValue(row, std::move(line), builder, lineType);
440440
}
441441
builder.EndList();
442442
return UpsertTValueBuffer(dbPath, builder).ExtractValueSync();
@@ -473,8 +473,8 @@ TStatus TImportFileClient::UpsertCsv(IInputStream& input, const TString& dbPath,
473473
progressCallback(readBytes, *inputSizeHint);
474474
}
475475

476-
auto asyncUpsertCSV = [&, buffer = std::move(buffer)]() mutable {
477-
return upsertCsv(std::move(buffer));
476+
auto asyncUpsertCSV = [&upsertCsv, row, buffer = std::move(buffer)]() mutable {
477+
return upsertCsv(row, std::move(buffer));
478478
};
479479

480480
batchBytes = 0;
@@ -489,7 +489,7 @@ TStatus TImportFileClient::UpsertCsv(IInputStream& input, const TString& dbPath,
489489
}
490490

491491
if (!buffer.empty() && countInput.Counter() > 0) {
492-
upsertCsv(std::move(buffer));
492+
upsertCsv(row, std::move(buffer));
493493
}
494494

495495
return WaitForQueue(0, inFlightRequests);
@@ -546,7 +546,7 @@ TStatus TImportFileClient::UpsertCsvByBlocks(const TString& filePath, const TStr
546546
builder.BeginList();
547547
for (auto&& line : buffer) {
548548
builder.AddListItem();
549-
parser.GetValue(std::move(line), builder, lineType);
549+
parser.GetValue(0, std::move(line), builder, lineType);
550550
}
551551
builder.EndList();
552552
return UpsertTValueBuffer(dbPath, builder);

0 commit comments

Comments
 (0)