Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix](json)Fix the bug that does not stop when reading json files #23062

Merged
merged 2 commits into from
Aug 18, 2023
Merged

[fix](json)Fix the bug that does not stop when reading json files #23062

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5bd43e1
[fix](json)Fix the bug that does not stop when reading json files
hubgeter Aug 16, 2023
060380c
[fix](json)Fix the bug that does not stop when reading json files
hubgeter Aug 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
[fix](json)Fix the bug that does not stop when reading json files
  • Loading branch information
@hubgeter
hubgeter committed Aug 16, 2023
commit 5bd43e1301dd208c66d36a2a16abaef7addaf5dc
1 change: 1 addition & 0 deletions be/src/vec/exec/format/json/new_json_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1054,6 +1054,7 @@ Status NewJsonReader::_read_one_message(std::unique_ptr<uint8_t[]>* file_buf, si
file_buf->reset(new uint8_t[file_size]);
Slice result(file_buf->get(), file_size);
RETURN_IF_ERROR(_file_reader->read_at(_current_offset, result, read_size, _io_ctx));
_current_offset += *read_size;
break;
}
case TFileType::FILE_STREAM: {
Expand Down
37 changes: 37 additions & 0 deletions regression-test/data/external_table_p0/tvf/test_hdfs_tvf.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,43 @@
8 chengdu 2345678
9 xian 2345679

-- !json_limit --
1 beijing 2345671
10 hefei 23456710
11 \N 23456711
12 hefei \N
2 shanghai 2345672
3 guangzhou 2345673
4 shenzhen 2345674
5 hangzhou 2345675
6 nanjing 2345676
7 wuhan 2345677
8 chengdu 2345678
9 xian 2345679

-- !json_limit --
1 beijing 1454547
10 hefei 2345676
2 shanghai 1244264
3 guangzhou 528369
4 shenzhen 594201
5 hangzhou 594201
6 nanjing 2345672
7 wuhan 2345673
8 chengdu 2345674
9 xian 2345675

-- !json_limit --
1 {"id": 1, "city": "beijing", "code": 2345671}
2 {"id": 2, "city": "shanghai", "code": 2345672}
3 {"id": 3, "city": "hangzhou", "code": 2345673}
4 {"id": 4, "city": "shenzhen", "code": 2345674}
5 {"id": 5, "city": "guangzhou", "code": 2345675}

-- !json_limit --
1 {"id": 1, "city": "beijing", "code": 2345671}
2 {"id": 2, "city": "shanghai", "code": 2345672}

-- !json_root --
1 beijing 2345671
2 shanghai 2345672
Expand Down
40 changes: 40 additions & 0 deletions regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,46 @@ suite("test_hdfs_tvf","external,hive,tvf,external_docker") {
"strip_outer_array" = "false",
"read_json_by_line" = "true") order by id; """


uri = "${defaultFS}" + "/user/doris/preinstalled_data/json_format_test/simple_object_json.json"
format = "json"
qt_json_limit """ select * from HDFS(
"uri" = "${uri}",
"fs.defaultFS"= "${defaultFS}",
"hadoop.username" = "${hdfsUserName}",
"format" = "${format}",
"strip_outer_array" = "false",
"read_json_by_line" = "true") order by id limit 100; """

uri = "${defaultFS}" + "/user/doris/preinstalled_data/json_format_test/one_array_json.json"
format = "json"
qt_json_limit """ select * from HDFS(
"uri" = "${uri}",
"fs.defaultFS"= "${defaultFS}",
"hadoop.username" = "${hdfsUserName}",
"format" = "${format}",
"strip_outer_array" = "true",
"read_json_by_line" = "false") order by id limit 100; """
uri = "${defaultFS}" + "/user/doris/preinstalled_data/json_format_test/nest_json.json"
format = "json"
qt_json_limit """ select * from HDFS(
"uri" = "${uri}",
"fs.defaultFS"= "${defaultFS}",
"hadoop.username" = "${hdfsUserName}",
"format" = "${format}",
"strip_outer_array" = "false",
"read_json_by_line" = "true") order by no limit 100; """
uri = "${defaultFS}" + "/user/doris/preinstalled_data/json_format_test/nest_json.json"
format = "json"
qt_json_limit """ select * from HDFS(
"uri" = "${uri}",
"fs.defaultFS"= "${defaultFS}",
"hadoop.username" = "${hdfsUserName}",
"format" = "${format}",
"strip_outer_array" = "false",
"read_json_by_line" = "true") order by no limit 2; """


// test json root
uri = "${defaultFS}" + "/user/doris/preinstalled_data/json_format_test/nest_json.json"
format = "json"
Expand Down