diff --git a/velox/serializers/CompactRowSerializer.cpp b/velox/serializers/CompactRowSerializer.cpp index a47be6c7e742..b44f804995ef 100644 --- a/velox/serializers/CompactRowSerializer.cpp +++ b/velox/serializers/CompactRowSerializer.cpp @@ -28,7 +28,7 @@ void CompactRowVectorSerde::estimateSerializedSize( } namespace { -class CompactRowVectorSerializer : public VectorSerializer { +class CompactRowVectorSerializer : public IterativeVectorSerializer { public: using TRowSize = uint32_t; @@ -120,7 +120,8 @@ std::string concatenatePartialRow( } // namespace -std::unique_ptr CompactRowVectorSerde::createSerializer( +std::unique_ptr +CompactRowVectorSerde::createIterativeSerializer( RowTypePtr /* type */, int32_t /* numRows */, StreamArena* streamArena, diff --git a/velox/serializers/CompactRowSerializer.h b/velox/serializers/CompactRowSerializer.h index 10c5a928cb49..8b6f1a998450 100644 --- a/velox/serializers/CompactRowSerializer.h +++ b/velox/serializers/CompactRowSerializer.h @@ -33,7 +33,7 @@ class CompactRowVectorSerde : public VectorSerde { // This method is not used in production code. It is only used to // support round-trip tests for deserialization. - std::unique_ptr createSerializer( + std::unique_ptr createIterativeSerializer( RowTypePtr type, int32_t numRows, StreamArena* streamArena, diff --git a/velox/serializers/PrestoSerializer.cpp b/velox/serializers/PrestoSerializer.cpp index a6e72af1b079..a4aa026c5eff 100644 --- a/velox/serializers/PrestoSerializer.cpp +++ b/velox/serializers/PrestoSerializer.cpp @@ -3248,9 +3248,9 @@ class PrestoBatchVectorSerializer : public BatchVectorSerializer { const std::unique_ptr codec_; }; -class PrestoVectorSerializer : public VectorSerializer { +class PrestoIterativeVectorSerializer : public IterativeVectorSerializer { public: - PrestoVectorSerializer( + PrestoIterativeVectorSerializer( const RowTypePtr& rowType, std::vector encodings, int32_t numRows, @@ -3281,7 +3281,7 @@ class PrestoVectorSerializer : public VectorSerializer { // Constructor that takes a row vector instead of only the types. This is // different because then we know exactly how each vector is encoded // (recursively). - PrestoVectorSerializer( + PrestoIterativeVectorSerializer( const RowVectorPtr& rowVector, StreamArena* streamArena, bool useLosslessTimestamp, @@ -3387,13 +3387,14 @@ void PrestoVectorSerde::estimateSerializedSize( estimateSerializedSizeInt(vector->loadedVector(), rows, sizes, scratch); } -std::unique_ptr PrestoVectorSerde::createSerializer( +std::unique_ptr +PrestoVectorSerde::createIterativeSerializer( RowTypePtr type, int32_t numRows, StreamArena* streamArena, const Options* options) { const auto prestoOptions = toPrestoOptions(options); - return std::make_unique( + return std::make_unique( type, prestoOptions.encodings, numRows, @@ -3416,7 +3417,7 @@ void PrestoVectorSerde::serializeEncoded( const Options* options, OutputStream* out) { auto prestoOptions = toPrestoOptions(options); - auto serializer = std::make_unique( + auto serializer = std::make_unique( vector, streamArena, prestoOptions.useLosslessTimestamp, diff --git a/velox/serializers/PrestoSerializer.h b/velox/serializers/PrestoSerializer.h index d5a79091b2fb..c170e92c49d1 100644 --- a/velox/serializers/PrestoSerializer.h +++ b/velox/serializers/PrestoSerializer.h @@ -24,10 +24,11 @@ namespace facebook::velox::serializer::presto { /// There are two ways to serialize data using PrestoVectorSerde: /// /// 1. In order to append multiple RowVectors into the same serialized payload, -/// one can first create a VectorSerializer using createSerializer(), then -/// append successive RowVectors using VectorSerializer::append(). In this case, -/// since different RowVector might encode columns differently, data is always -/// flattened in the serialized payload. +/// one can first create an IterativeVectorSerializer using +/// createIterativeSerializer(), then append successive RowVectors using +/// IterativeVectorSerializer::append(). In this case, since different RowVector +/// might encode columns differently, data is always flattened in the serialized +/// payload. /// /// Note that there are two flavors of append(), one that takes a range of rows, /// and one that takes a list of row ids. The former is useful when serializing @@ -77,7 +78,7 @@ class PrestoVectorSerde : public VectorSerde { vector_size_t** sizes, Scratch& scratch) override; - std::unique_ptr createSerializer( + std::unique_ptr createIterativeSerializer( RowTypePtr type, int32_t numRows, StreamArena* streamArena, diff --git a/velox/serializers/UnsafeRowSerializer.cpp b/velox/serializers/UnsafeRowSerializer.cpp index 342311c5d926..0d940000fd70 100644 --- a/velox/serializers/UnsafeRowSerializer.cpp +++ b/velox/serializers/UnsafeRowSerializer.cpp @@ -29,7 +29,7 @@ void UnsafeRowVectorSerde::estimateSerializedSize( } namespace { -class UnsafeRowVectorSerializer : public VectorSerializer { +class UnsafeRowVectorSerializer : public IterativeVectorSerializer { public: using TRowSize = uint32_t; @@ -122,7 +122,8 @@ std::string concatenatePartialRow( } // namespace -std::unique_ptr UnsafeRowVectorSerde::createSerializer( +std::unique_ptr +UnsafeRowVectorSerde::createIterativeSerializer( RowTypePtr /* type */, int32_t /* numRows */, StreamArena* streamArena, diff --git a/velox/serializers/UnsafeRowSerializer.h b/velox/serializers/UnsafeRowSerializer.h index 1bf41c4f0cae..1c793c98717e 100644 --- a/velox/serializers/UnsafeRowSerializer.h +++ b/velox/serializers/UnsafeRowSerializer.h @@ -31,7 +31,7 @@ class UnsafeRowVectorSerde : public VectorSerde { // This method is not used in production code. It is only used to // support round-trip tests for deserialization. - std::unique_ptr createSerializer( + std::unique_ptr createIterativeSerializer( RowTypePtr type, int32_t numRows, StreamArena* streamArena, diff --git a/velox/serializers/tests/CompactRowSerializerTest.cpp b/velox/serializers/tests/CompactRowSerializerTest.cpp index a556c1e538ab..50fcd8d2c91f 100644 --- a/velox/serializers/tests/CompactRowSerializerTest.cpp +++ b/velox/serializers/tests/CompactRowSerializerTest.cpp @@ -43,7 +43,8 @@ class CompactRowSerializerTest : public ::testing::Test, auto arena = std::make_unique(pool_.get()); auto rowType = asRowType(rowVector->type()); - auto serializer = serde_->createSerializer(rowType, numRows, arena.get()); + auto serializer = + serde_->createIterativeSerializer(rowType, numRows, arena.get()); Scratch scratch; serializer->append(rowVector, folly::Range(rows.data(), numRows), scratch); diff --git a/velox/serializers/tests/PrestoSerializerTest.cpp b/velox/serializers/tests/PrestoSerializerTest.cpp index 230842b43a87..cc149ff646fa 100644 --- a/velox/serializers/tests/PrestoSerializerTest.cpp +++ b/velox/serializers/tests/PrestoSerializerTest.cpp @@ -90,8 +90,8 @@ class PrestoSerializerTest auto rowType = asRowType(rowVector->type()); auto numRows = rowVector->size(); auto paramOptions = getParamSerdeOptions(serdeOptions); - auto serializer = - serde_->createSerializer(rowType, numRows, arena.get(), ¶mOptions); + auto serializer = serde_->createIterativeSerializer( + rowType, numRows, arena.get(), ¶mOptions); vector_size_t sizeEstimate = 0; Scratch scratch; diff --git a/velox/serializers/tests/UnsafeRowSerializerTest.cpp b/velox/serializers/tests/UnsafeRowSerializerTest.cpp index a8cc0d4c2773..ce6ad23aa1f5 100644 --- a/velox/serializers/tests/UnsafeRowSerializerTest.cpp +++ b/velox/serializers/tests/UnsafeRowSerializerTest.cpp @@ -43,7 +43,8 @@ class UnsafeRowSerializerTest : public ::testing::Test, auto arena = std::make_unique(pool_.get()); auto rowType = std::dynamic_pointer_cast(rowVector->type()); - auto serializer = serde_->createSerializer(rowType, numRows, arena.get()); + auto serializer = + serde_->createIterativeSerializer(rowType, numRows, arena.get()); Scratch scratch; serializer->append(rowVector, folly::Range(rows.data(), numRows), scratch); diff --git a/velox/vector/VectorStream.cpp b/velox/vector/VectorStream.cpp index 10dc62edcddf..117f382d6ffd 100644 --- a/velox/vector/VectorStream.cpp +++ b/velox/vector/VectorStream.cpp @@ -19,7 +19,7 @@ namespace facebook::velox { -void VectorSerializer::append(const RowVectorPtr& vector) { +void IterativeVectorSerializer::append(const RowVectorPtr& vector) { const IndexRange allRows{0, vector->size()}; Scratch scratch; append(vector, folly::Range(&allRows, 1), scratch); @@ -33,8 +33,8 @@ void BatchVectorSerializer::serialize( } namespace { -/// An adapter class that can be used to convert a VectorSerializer into a -/// BatchVectorSerializer for VectorSerdes that don't want to implement a +/// An adapter class that can be used to convert an IterativeVectorSerializer +/// into a BatchVectorSerializer for VectorSerdes that don't want to implement a /// separate serializer. class BatchVectorSerializerAdapter : public BatchVectorSerializer { public: @@ -55,7 +55,7 @@ class BatchVectorSerializerAdapter : public BatchVectorSerializer { } StreamArena arena(pool_); - auto serializer = serde_->createSerializer( + auto serializer = serde_->createIterativeSerializer( asRowType(vector->type()), numRows, &arena, options_); serializer->append(vector, ranges, scratch); serializer->flush(stream); @@ -145,7 +145,7 @@ void VectorStreamGroup::createStreamTree( RowTypePtr type, int32_t numRows, const VectorSerde::Options* options) { - serializer_ = serde_->createSerializer(type, numRows, this, options); + serializer_ = serde_->createIterativeSerializer(type, numRows, this, options); } void VectorStreamGroup::append( diff --git a/velox/vector/VectorStream.h b/velox/vector/VectorStream.h index d47e7af36713..544c2934dd6c 100644 --- a/velox/vector/VectorStream.h +++ b/velox/vector/VectorStream.h @@ -39,9 +39,9 @@ struct IndexRange { /// Use successive calls to `append` to add more rows to the serialization /// buffer. Then call `flush` to write the aggregate serialized data to an /// OutputStream. -class VectorSerializer { +class IterativeVectorSerializer { public: - virtual ~VectorSerializer() = default; + virtual ~IterativeVectorSerializer() = default; /// Serialize a subset of rows in a vector. virtual void append( @@ -153,7 +153,18 @@ class VectorSerde { estimateSerializedSize(vector, ranges, sizes, scratch); } - virtual std::unique_ptr createSerializer( + // TODO: Remove createSerializer once Presto is updated to call + // createIterativeSerializer. + virtual std::unique_ptr createSerializer( + RowTypePtr type, + int32_t numRows, + StreamArena* streamArena, + const Options* options = nullptr) { + return createIterativeSerializer( + std::move(type), numRows, streamArena, options); + } + + virtual std::unique_ptr createIterativeSerializer( RowTypePtr type, int32_t numRows, StreamArena* streamArena, @@ -287,7 +298,7 @@ class VectorStreamGroup : public StreamArena { const VectorSerde::Options* options = nullptr); private: - std::unique_ptr serializer_; + std::unique_ptr serializer_; VectorSerde* serde_{nullptr}; }; diff --git a/velox/vector/tests/VectorStreamTest.cpp b/velox/vector/tests/VectorStreamTest.cpp index ecc9f8cda2e0..a5b6707163fd 100644 --- a/velox/vector/tests/VectorStreamTest.cpp +++ b/velox/vector/tests/VectorStreamTest.cpp @@ -25,7 +25,7 @@ class MockVectorSerde : public VectorSerde { const folly::Range& ranges, vector_size_t** sizes) override {} - std::unique_ptr createSerializer( + std::unique_ptr createIterativeSerializer( RowTypePtr type, int32_t numRows, StreamArena* streamArena,