Skip to content

YQL-17371: fix ASCII-compatibe bytea output in pgrun #1356

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 26, 2024
Merged

YQL-17371: fix ASCII-compatibe bytea output in pgrun #1356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4293827
Fixed bytea output if it is ASCII-compatible
marsaly79 Jan 26, 2024
9202a9c
Updated regression tests' status
marsaly79 Jan 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions ydb/library/yql/tests/postgresql/cases/float4.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,15 @@ SELECT '-9223372036854775808.5'::float4::int8;

SELECT '-9223380000000000000'::float4::int8;
ERROR: bigint out of range
-- Test for correct input rounding in edge cases.
-- These lists are from Paxson 1991, excluding subnormals and
-- inputs of over 9 sig. digits.
SELECT float4send('5e-20'::float4);
float4send
------------
\x1f6c1e4a
(1 row)

SELECT float4send('67e14'::float4);
float4send
------------
Expand Down Expand Up @@ -350,6 +359,12 @@ SELECT float4send('702990899e-20'::float4);
\x2cf757ca
(1 row)

SELECT float4send('3e-23'::float4);
float4send
------------
\x1a111234
(1 row)

SELECT float4send('57e18'::float4);
float4send
------------
Expand Down Expand Up @@ -392,6 +407,12 @@ SELECT float4send('82381273e-35'::float4);
\x128289d1
(1 row)

SELECT float4send('750486563e-38'::float4);
float4send
------------
\x0f18377e
(1 row)

-- Test that the smallest possible normalized input value inputs
-- correctly, either in 9-significant-digit or shortest-decimal
-- format.
Expand Down
6 changes: 6 additions & 0 deletions ydb/library/yql/tests/postgresql/cases/float4.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,18 +76,24 @@ SELECT '9223369837831520256'::float4::int8;
SELECT '9223372036854775807'::float4::int8;
SELECT '-9223372036854775808.5'::float4::int8;
SELECT '-9223380000000000000'::float4::int8;
-- Test for correct input rounding in edge cases.
-- These lists are from Paxson 1991, excluding subnormals and
-- inputs of over 9 sig. digits.
SELECT float4send('5e-20'::float4);
SELECT float4send('67e14'::float4);
SELECT float4send('985e15'::float4);
SELECT float4send('55895e-16'::float4);
SELECT float4send('7038531e-32'::float4);
SELECT float4send('702990899e-20'::float4);
SELECT float4send('3e-23'::float4);
SELECT float4send('57e18'::float4);
SELECT float4send('789e-35'::float4);
SELECT float4send('2539e-18'::float4);
SELECT float4send('76173e28'::float4);
SELECT float4send('887745e-11'::float4);
SELECT float4send('5382571e-37'::float4);
SELECT float4send('82381273e-35'::float4);
SELECT float4send('750486563e-38'::float4);
-- Test that the smallest possible normalized input value inputs
-- correctly, either in 9-significant-digit or shortest-decimal
-- format.
Expand Down
7 changes: 7 additions & 0 deletions ydb/library/yql/tests/postgresql/cases/float8.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ SELECT '-10e-400'::float8;
ERROR: "-10e-400" is out of range for type double precision
LINE 1: SELECT '-10e-400'::float8;
^
-- test smallest normalized input
SELECT float8send('2.2250738585072014E-308'::float8);
float8send
--------------------
\x0010000000000000
(1 row)

-- bad input
INSERT INTO FLOAT8_TBL(f1) VALUES ('');
ERROR: invalid input syntax for type double precision: ""
Expand Down
2 changes: 2 additions & 0 deletions ydb/library/yql/tests/postgresql/cases/float8.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ SELECT '10e400'::float8;
SELECT '-10e400'::float8;
SELECT '10e-400'::float8;
SELECT '-10e-400'::float8;
-- test smallest normalized input
SELECT float8send('2.2250738585072014E-308'::float8);
-- bad input
INSERT INTO FLOAT8_TBL(f1) VALUES ('');
INSERT INTO FLOAT8_TBL(f1) VALUES (' ');
Expand Down
30 changes: 30 additions & 0 deletions ydb/library/yql/tests/postgresql/cases/strings.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,30 @@ SELECT E'\\xDe00BeEf'::bytea;
\xde00beef
(1 row)

SELECT E'DeAdBeEf'::bytea;
bytea
--------------------
\x4465416442654566
(1 row)

SELECT E'De\\000dBeEf'::bytea;
bytea
--------------------
\x4465006442654566
(1 row)

SELECT E'De\123dBeEf'::bytea;
bytea
--------------------
\x4465536442654566
(1 row)

SELECT E'De\\123dBeEf'::bytea;
bytea
--------------------
\x4465536442654566
(1 row)

SELECT E'De\\678dBeEf'::bytea;
ERROR: invalid input syntax for type bytea
LINE 1: SELECT E'De\\678dBeEf'::bytea;
Expand Down Expand Up @@ -207,6 +231,12 @@ SELECT E'DeAdBeEf'::bytea;
DeAdBeEf
(1 row)

SELECT E'De\\000dBeEf'::bytea;
bytea
-------------
De\000dBeEf
(1 row)

SELECT E'De\\123dBeEf'::bytea;
bytea
----------
Expand Down
5 changes: 5 additions & 0 deletions ydb/library/yql/tests/postgresql/cases/strings.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,17 @@ SELECT E'\\x De Ad Be Ef '::bytea;
SELECT E'\\xDeAdBeE'::bytea;
SELECT E'\\xDeAdBeEx'::bytea;
SELECT E'\\xDe00BeEf'::bytea;
SELECT E'DeAdBeEf'::bytea;
SELECT E'De\\000dBeEf'::bytea;
SELECT E'De\123dBeEf'::bytea;
SELECT E'De\\123dBeEf'::bytea;
SELECT E'De\\678dBeEf'::bytea;
SET bytea_output TO escape;
SELECT E'\\xDeAdBeEf'::bytea;
SELECT E'\\x De Ad Be Ef '::bytea;
SELECT E'\\xDe00BeEf'::bytea;
SELECT E'DeAdBeEf'::bytea;
SELECT E'De\\000dBeEf'::bytea;
SELECT E'De\\123dBeEf'::bytea;
SELECT CAST(name 'namefield' AS text) AS "text(name)";
SELECT CAST(name 'namefield' AS char(10)) AS "char(name)";
Expand Down
6 changes: 3 additions & 3 deletions ydb/library/yql/tests/postgresql/pg_tests.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ date,264,200,75.76
dbsize,24,24,100.0
delete,10,0,0.0
expressions,63,14,22.22
float4,96,80,83.33
float8,168,132,78.57
float4,96,82,85.42
float8,168,140,83.33
functional_deps,40,7,17.5
horology,306,79,25.82
insert,357,15,4.2
Expand All @@ -40,7 +40,7 @@ select_distinct_on,4,0,0.0
select_having,23,16,69.57
select_implicit,44,13,29.55
select_into,67,3,4.48
strings,390,81,20.77
strings,390,86,22.05
subselect,234,5,2.14
text,76,16,21.05
time,39,33,84.62
Expand Down
6 changes: 3 additions & 3 deletions ydb/library/yql/tests/postgresql/status.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@
|| 7 | int4 | 70 | 70 | 100.0 | 29.09.2023 | ||
|| 8 | int8 | 142 | 113 (+65) | 79.58 | 22.01.2024 | YQL-17614 ||
|| 9 | oid | 27 | 27 (+6) | 100.0 | 22.01.2024 | YQL-17623 ||
|| 10 | float4 | 96 | 80 (+32) | 83.33 | 23.01.2024 | YQL-17586 ||
|| 11 | float8 | 168 | 132 (+36) | 78.57 | 23.01.2024 | YQL-17628 ||
|| 10 | float4 | 96 | 82 (+2) | 85.42 | 26.01.2024 | YQL-17586 ||
|| 11 | float8 | 168 | 140 (+8) | 83.33 | 26.01.2024 | YQL-17628 ||
|| 12 | bit | 115 | 84 | 73.04 | 12.12.2023 | YQL-17634 ||
|| 13 | numeric | 915 | 721 (+6) | 78.8 | 23.01.2024 | YQL-17629 ||
|| 14 | uuid | 36 | 0 | 0.0 | 02.05.2023 | YQL-17636 ||
|| 15 | strings | 390 | 81 (+50) | 20.77 | 24.01.2024 | YQL-17587 ||
|| 15 | strings | 390 | 86 (+5) | 22.05 | 26.01.2024 | YQL-17587 ||
|| 16 | numerology | 24 | 8 | 33.33 | 26.07.2023 | ||
|| 17 | date | 264 | 200 | 75.76 | 12.12.2023 | ||
|| 18 | time | 39 | 33 | 84.62 | 12.12.2023 | ||
Expand Down
4 changes: 2 additions & 2 deletions ydb/library/yql/tests/postgresql/status.old
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@
|| 7 | int4 | 70 | 70 | 100.0 | 29.09.2023 | ||
|| 8 | int8 | 142 | 113 (+65) | 79.58 | 22.01.2024 | YQL-17614 ||
|| 9 | oid | 27 | 27 (+6) | 100.0 | 22.01.2024 | YQL-17623 ||
|| 10 | float4 | 96 | 80 (+14) | 83.33 | 23.01.2024 | YQL-17586 ||
|| 10 | float4 | 96 | 80 (+32) | 83.33 | 23.01.2024 | YQL-17586 ||
|| 11 | float8 | 168 | 132 (+36) | 78.57 | 23.01.2024 | YQL-17628 ||
|| 12 | bit | 115 | 84 | 73.04 | 12.12.2023 | YQL-17634 ||
|| 13 | numeric | 915 | 721 (+6) | 78.8 | 23.01.2024 | YQL-17629 ||
|| 14 | uuid | 36 | 0 | 0.0 | 02.05.2023 | YQL-17636 ||
|| 15 | strings | 390 | 76 (+45) | 19.49 | 23.01.2024 | YQL-17587 ||
|| 15 | strings | 390 | 81 (+50) | 20.77 | 24.01.2024 | YQL-17587 ||
|| 16 | numerology | 24 | 8 | 33.33 | 26.07.2023 | ||
|| 17 | date | 264 | 200 | 75.76 | 12.12.2023 | ||
|| 18 | time | 39 | 33 | 84.62 | 12.12.2023 | ||
Expand Down
87 changes: 45 additions & 42 deletions ydb/library/yql/tools/pgrun/pgrun.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -815,54 +815,57 @@ std::string FormatCell(const TString& data, const TColumn& column, size_t index,

TString GetCellData(const NYT::TNode& cell, const TColumn& column) {
if (column.Type == "bytea") {
if (cell.IsList()) {
TString result;
const auto rawValue = (cell.IsList())
? Base64Decode(cell.AsList()[0].AsString())
: cell.AsString();

const auto rawValue = Base64Decode(cell.AsList()[0].AsString());
switch (byteaOutput) {
case EByteaOutput::hex: {
const auto expectedSize = rawValue.size() * 2 + 2;
result.resize(expectedSize);
result[0] = '\\';
result[1] = 'x';
const auto cnt = HexEncode(rawValue.data(), rawValue.size(), result.begin() + 2);
switch (byteaOutput) {
case EByteaOutput::hex: {
TString result;

Y_ASSERT(cnt + 2 == expectedSize);
const auto expectedSize = rawValue.size() * 2 + 2;
result.resize(expectedSize);
result[0] = '\\';
result[1] = 'x';
const auto cnt = HexEncode(rawValue.data(), rawValue.size(), result.begin() + 2);

return result;
}
case EByteaOutput::escape: {
ui64 expectedSize = std::accumulate(rawValue.cbegin(), rawValue.cend(), 0U,
[] (ui64 acc, char c) {
return acc + ((c == '\\')
? 2
: ((ui8)c < 0x20 || 0x7e < (ui8)c)
? 4
: 1);
});
result.resize(expectedSize);
auto p = result.begin();
for (const auto c : rawValue) {
if (c == '\\') {
*p++ = '\\';
*p++ = '\\';
} else if ((ui8)c < 0x20 || 0x7e < (ui8)c) {
auto val = (ui8)c;

*p++ = '\\';
*p++ = ((val >> 6) & 03) + '0';
*p++ = ((val >> 3) & 07) + '0';
*p++ = (val & 07) + '0';
} else {
*p++ = c;
}
}
Y_ASSERT(cnt + 2 == expectedSize);

return result;
return result;
}
case EByteaOutput::escape: {
TString result;

ui64 expectedSize = std::accumulate(rawValue.cbegin(), rawValue.cend(), 0U,
[] (ui64 acc, char c) {
return acc + ((c == '\\')
? 2
: ((ui8)c < 0x20 || 0x7e < (ui8)c)
Copy link
Collaborator

@MrLolthe1st MrLolthe1st Jan 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

давай тут magic уберем и вынесем куда-нибудь (можно в макрос/constexpr функцию)

Copy link
Contributor Author

@marsaly79 marsaly79 Jan 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Это взято из кода PG. Можно поправить, но, в целом, стандартная штука же

? 4
: 1);
});
result.resize(expectedSize);
auto p = result.begin();
for (const auto c : rawValue) {
if (c == '\\') {
*p++ = '\\';
*p++ = '\\';
} else if ((ui8)c < 0x20 || 0x7e < (ui8)c) {
Copy link
Collaborator

@MrLolthe1st MrLolthe1st Jan 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

тот же самый magic

auto val = (ui8)c;

*p++ = '\\';
*p++ = ((val >> 6) & 03) + '0';
*p++ = ((val >> 3) & 07) + '0';
*p++ = (val & 07) + '0';
} else {
*p++ = c;
}
}
default:
throw yexception() << "Unhandled EByteaOutput value";

return result;
}
default:
throw yexception() << "Unhandled EByteaOutput value";
}
}
return cell.AsString();
Expand Down