Skip to content

GH-44769: [C++][Parquet] Fix read/write of metadata length footer on big-endian systems #44787

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mapleFU merged 4 commits into from
Nov 20, 2024
Merged

GH-44769: [C++][Parquet] Fix read/write of metadata length footer on big-endian systems #44787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fd3cf89
Fix writing Parquet metadata length footer
QuLogic Nov 19, 2024
4b1dd1b
Fix reading Parquet metadata length footer
QuLogic Nov 19, 2024
d132d20
Fix linting
QuLogic Nov 19, 2024
3c44c11
Use a temporary for little-endian values
QuLogic Nov 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions cpp/src/parquet/file_reader.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,9 +497,10 @@ class SerializedFile : public ParquetFileReader::Contents {
"is not a parquet file.");
}
// Both encrypted/unencrypted footers have the same footer length check.
uint32_t metadata_len = ::arrow::util::SafeLoadAs<uint32_t>(
reinterpret_cast<const uint8_t*>(footer_buffer->data()) + footer_read_size -
kFooterSize);
uint32_t metadata_len =
::arrow::bit_util::FromLittleEndian(::arrow::util::SafeLoadAs<uint32_t>(
reinterpret_cast<const uint8_t*>(footer_buffer->data()) + footer_read_size -
kFooterSize));
if (metadata_len > source_size_ - kFooterSize) {
throw ParquetInvalidOrCorruptedFileException(
"Parquet file size is ", source_size_,
Expand Down
14 changes: 11 additions & 3 deletions cpp/src/parquet/file_writer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -426,8 +426,10 @@ class FileSerializer : public ParquetFileWriter::Contents {
WriteEncryptedFileMetadata(*file_metadata_, sink_.get(), footer_encryptor, true);
PARQUET_ASSIGN_OR_THROW(position, sink_->Tell());
uint32_t footer_and_crypto_len = static_cast<uint32_t>(position - metadata_start);
uint32_t footer_and_crypto_len_le =
::arrow::bit_util::ToLittleEndian(footer_and_crypto_len);
PARQUET_THROW_NOT_OK(
sink_->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len), 4));
sink_->Write(reinterpret_cast<uint8_t*>(&footer_and_crypto_len_le), 4));
PARQUET_THROW_NOT_OK(sink_->Write(kParquetEMagic, 4));
} else { // Encrypted file with plaintext footer
file_metadata_ = metadata_->Finish(key_value_metadata_);
Expand Down Expand Up @@ -539,7 +541,10 @@ void WriteFileMetaData(const FileMetaData& file_metadata, ArrowOutputStream* sin
metadata_len = static_cast<uint32_t>(position) - metadata_len;

// Write Footer
PARQUET_THROW_NOT_OK(sink->Write(reinterpret_cast<uint8_t*>(&metadata_len), 4));
{
uint32_t metadata_len_le = ::arrow::bit_util::ToLittleEndian(metadata_len);
PARQUET_THROW_NOT_OK(sink->Write(reinterpret_cast<uint8_t*>(&metadata_len_le), 4));
}
PARQUET_THROW_NOT_OK(sink->Write(kParquetMagic, 4));
}

Expand All @@ -562,7 +567,10 @@ void WriteEncryptedFileMetadata(const FileMetaData& file_metadata,
PARQUET_ASSIGN_OR_THROW(position, sink->Tell());
metadata_len = static_cast<uint32_t>(position) - metadata_len;

PARQUET_THROW_NOT_OK(sink->Write(reinterpret_cast<uint8_t*>(&metadata_len), 4));
{
uint32_t metadata_len_le = ::arrow::bit_util::ToLittleEndian(metadata_len);
PARQUET_THROW_NOT_OK(sink->Write(reinterpret_cast<uint8_t*>(&metadata_len_le), 4));
}
PARQUET_THROW_NOT_OK(sink->Write(kParquetMagic, 4));
}
}
Expand Down
Loading