Skip to content

Commit c2a480b

Browse files
ivanmorozov333ivanmorozov333
committed
Revert "Stable-24-1 patch for cs (ydb-platform#2142)"
This reverts commit 288a80b.
1 parent 14f67b4 commit c2a480b

File tree

141 files changed

+1035
-1262
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

141 files changed

+1035
-1262
lines changed

‎ydb/core/formats/arrow/arrow_filter.cpp‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -569,7 +569,7 @@ TColumnFilter TColumnFilter::CombineSequentialAnd(const TColumnFilter& extFilter
569569
++itExt;
570570
}
571571
}
572-
AFL_VERIFY(itSelf == Filter.end() && itExt == extFilter.Filter.cend());
572+
Y_ABORT_UNLESS(itSelf == Filter.end() && itExt == extFilter.Filter.cend());
573573
TColumnFilter result = TColumnFilter::BuildAllowFilter();
574574
std::swap(resultFilter, result.Filter);
575575
std::swap(curCurrent, result.LastValue);
@@ -611,12 +611,4 @@ std::optional<ui32> TColumnFilter::GetFilteredCount() const {
611611
return *FilteredCount;
612612
}
613613

614-
void TColumnFilter::Append(const TColumnFilter& filter) {
615-
bool currentVal = filter.GetStartValue();
616-
for (auto&& i : filter.Filter) {
617-
Add(currentVal, i);
618-
currentVal = !currentVal;
619-
}
620-
}
621-
622614
}

‎ydb/core/formats/arrow/arrow_filter.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ class TColumnFilter {
5252
FilteredCount.reset();
5353
}
5454
public:
55-
void Append(const TColumnFilter& filter);
5655
void Add(const bool value, const ui32 count = 1);
5756
std::optional<ui32> GetFilteredCount() const;
5857
const std::vector<bool>& BuildSimpleFilter() const;

‎ydb/core/formats/arrow/arrow_helpers.cpp‎

Lines changed: 3 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#include "common/validation.h"
55
#include "merging_sorted_input_stream.h"
66
#include "permutations.h"
7-
#include "serializer/native.h"
7+
#include "serializer/batch_only.h"
88
#include "serializer/abstract.h"
99
#include "serializer/stream.h"
1010
#include "simple_arrays_cache.h"
@@ -106,7 +106,7 @@ std::shared_ptr<arrow::Schema> DeserializeSchema(const TString& str) {
106106
}
107107

108108
TString SerializeBatch(const std::shared_ptr<arrow::RecordBatch>& batch, const arrow::ipc::IpcWriteOptions& options) {
109-
return NSerialization::TNativeSerializer(options).SerializePayload(batch);
109+
return NSerialization::TBatchPayloadSerializer(options).Serialize(batch);
110110
}
111111

112112
TString SerializeBatchNoCompression(const std::shared_ptr<arrow::RecordBatch>& batch) {
@@ -117,7 +117,7 @@ TString SerializeBatchNoCompression(const std::shared_ptr<arrow::RecordBatch>& b
117117

118118
std::shared_ptr<arrow::RecordBatch> DeserializeBatch(const TString& blob, const std::shared_ptr<arrow::Schema>& schema)
119119
{
120-
auto result = NSerialization::TNativeSerializer().Deserialize(blob, schema);
120+
auto result = NSerialization::TBatchPayloadDeserializer(schema).Deserialize(blob);
121121
if (result.ok()) {
122122
return *result;
123123
} else {
@@ -977,35 +977,4 @@ std::shared_ptr<arrow::RecordBatch> MergeColumns(const std::vector<std::shared_p
977977
return arrow::RecordBatch::Make(schema, *recordsCount, columns);
978978
}
979979

980-
std::vector<std::shared_ptr<arrow::RecordBatch>> SliceToRecordBatches(const std::shared_ptr<arrow::Table>& t) {
981-
std::set<ui32> splitPositions;
982-
const ui32 numRows = t->num_rows();
983-
for (auto&& i : t->columns()) {
984-
ui32 pos = 0;
985-
for (auto&& arr : i->chunks()) {
986-
splitPositions.emplace(pos);
987-
pos += arr->length();
988-
}
989-
AFL_VERIFY(pos == t->num_rows());
990-
}
991-
std::vector<std::vector<std::shared_ptr<arrow::Array>>> slicedData;
992-
slicedData.resize(splitPositions.size());
993-
std::vector<ui32> positions(splitPositions.begin(), splitPositions.end());
994-
for (auto&& i : t->columns()) {
995-
for (ui32 idx = 0; idx < positions.size(); ++idx) {
996-
auto slice = i->Slice(positions[idx], ((idx + 1 == positions.size()) ? numRows : positions[idx + 1]) - positions[idx]);
997-
AFL_VERIFY(slice->num_chunks() == 1);
998-
slicedData[idx].emplace_back(slice->chunks().front());
999-
}
1000-
}
1001-
std::vector<std::shared_ptr<arrow::RecordBatch>> result;
1002-
ui32 count = 0;
1003-
for (auto&& i : slicedData) {
1004-
result.emplace_back(arrow::RecordBatch::Make(t->schema(), i.front()->length(), i));
1005-
count += result.back()->num_rows();
1006-
}
1007-
AFL_VERIFY(count == t->num_rows())("count", count)("t", t->num_rows());
1008-
return result;
1009-
}
1010-
1011980
}

‎ydb/core/formats/arrow/arrow_helpers.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,6 @@ inline bool HasNulls(const std::shared_ptr<arrow::Array>& column) {
128128
return column->null_bitmap_data();
129129
}
130130

131-
std::vector<std::shared_ptr<arrow::RecordBatch>> SliceToRecordBatches(const std::shared_ptr<arrow::Table>& t);
132-
133131
bool ArrayScalarsEqual(const std::shared_ptr<arrow::Array>& lhs, const std::shared_ptr<arrow::Array>& rhs);
134132
std::shared_ptr<arrow::Array> BoolVecToArray(const std::vector<bool>& vec);
135133

‎ydb/core/formats/arrow/compression/CMakeLists.darwin-arm64.txt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
2+
# This file was generated by the build system used internally in the Yandex monorepo.
3+
# Only simple modifications are allowed (adding source-files to targets, adding simple properties
4+
# like target_include_directories). These modifications will be ported to original
5+
# ya.make files by maintainers. Any complex modifications which can't be ported back to the
6+
# original buildsystem will not be accepted.
7+
8+
9+
10+
add_library(formats-arrow-compression)
11+
target_link_libraries(formats-arrow-compression PUBLIC
12+
contrib-libs-cxxsupp
13+
yutil
14+
libs-apache-arrow
15+
ydb-core-protos
16+
core-formats-arrow
17+
ydb-library-conclusion
18+
)
19+
target_sources(formats-arrow-compression PRIVATE
20+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/diff.cpp
21+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/object.cpp
22+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/parsing.cpp
23+
)

‎ydb/core/formats/arrow/compression/CMakeLists.darwin-x86_64.txt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
2+
# This file was generated by the build system used internally in the Yandex monorepo.
3+
# Only simple modifications are allowed (adding source-files to targets, adding simple properties
4+
# like target_include_directories). These modifications will be ported to original
5+
# ya.make files by maintainers. Any complex modifications which can't be ported back to the
6+
# original buildsystem will not be accepted.
7+
8+
9+
10+
add_library(formats-arrow-compression)
11+
target_link_libraries(formats-arrow-compression PUBLIC
12+
contrib-libs-cxxsupp
13+
yutil
14+
libs-apache-arrow
15+
ydb-core-protos
16+
core-formats-arrow
17+
ydb-library-conclusion
18+
)
19+
target_sources(formats-arrow-compression PRIVATE
20+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/diff.cpp
21+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/object.cpp
22+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/parsing.cpp
23+
)

‎ydb/core/formats/arrow/compression/CMakeLists.linux-aarch64.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
2+
# This file was generated by the build system used internally in the Yandex monorepo.
3+
# Only simple modifications are allowed (adding source-files to targets, adding simple properties
4+
# like target_include_directories). These modifications will be ported to original
5+
# ya.make files by maintainers. Any complex modifications which can't be ported back to the
6+
# original buildsystem will not be accepted.
7+
8+
9+
10+
add_library(formats-arrow-compression)
11+
target_link_libraries(formats-arrow-compression PUBLIC
12+
contrib-libs-linux-headers
13+
contrib-libs-cxxsupp
14+
yutil
15+
libs-apache-arrow
16+
ydb-core-protos
17+
core-formats-arrow
18+
ydb-library-conclusion
19+
)
20+
target_sources(formats-arrow-compression PRIVATE
21+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/diff.cpp
22+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/object.cpp
23+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/parsing.cpp
24+
)

‎ydb/core/formats/arrow/compression/CMakeLists.linux-x86_64.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
2+
# This file was generated by the build system used internally in the Yandex monorepo.
3+
# Only simple modifications are allowed (adding source-files to targets, adding simple properties
4+
# like target_include_directories). These modifications will be ported to original
5+
# ya.make files by maintainers. Any complex modifications which can't be ported back to the
6+
# original buildsystem will not be accepted.
7+
8+
9+
10+
add_library(formats-arrow-compression)
11+
target_link_libraries(formats-arrow-compression PUBLIC
12+
contrib-libs-linux-headers
13+
contrib-libs-cxxsupp
14+
yutil
15+
libs-apache-arrow
16+
ydb-core-protos
17+
core-formats-arrow
18+
ydb-library-conclusion
19+
)
20+
target_sources(formats-arrow-compression PRIVATE
21+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/diff.cpp
22+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/object.cpp
23+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/parsing.cpp
24+
)

‎ydb/core/formats/arrow/compression/CMakeLists.txt‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
2+
# This file was generated by the build system used internally in the Yandex monorepo.
3+
# Only simple modifications are allowed (adding source-files to targets, adding simple properties
4+
# like target_include_directories). These modifications will be ported to original
5+
# ya.make files by maintainers. Any complex modifications which can't be ported back to the
6+
# original buildsystem will not be accepted.
7+
8+
9+
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
10+
include(CMakeLists.linux-x86_64.txt)
11+
elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
12+
include(CMakeLists.linux-aarch64.txt)
13+
elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
14+
include(CMakeLists.darwin-x86_64.txt)
15+
elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
16+
include(CMakeLists.darwin-arm64.txt)
17+
elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
18+
include(CMakeLists.windows-x86_64.txt)
19+
endif()

‎ydb/core/formats/arrow/compression/CMakeLists.windows-x86_64.txt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
2+
# This file was generated by the build system used internally in the Yandex monorepo.
3+
# Only simple modifications are allowed (adding source-files to targets, adding simple properties
4+
# like target_include_directories). These modifications will be ported to original
5+
# ya.make files by maintainers. Any complex modifications which can't be ported back to the
6+
# original buildsystem will not be accepted.
7+
8+
9+
10+
add_library(formats-arrow-compression)
11+
target_link_libraries(formats-arrow-compression PUBLIC
12+
contrib-libs-cxxsupp
13+
yutil
14+
libs-apache-arrow
15+
ydb-core-protos
16+
core-formats-arrow
17+
ydb-library-conclusion
18+
)
19+
target_sources(formats-arrow-compression PRIVATE
20+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/diff.cpp
21+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/object.cpp
22+
${CMAKE_SOURCE_DIR}/ydb/core/formats/arrow/compression/parsing.cpp
23+
)

0 commit comments

Comments
 (0)