Skip to content

Commit

Permalink
PARQUET-1630: add empty compression union for Bloom filter (#149)
Browse files Browse the repository at this point in the history
Right now no compression methods are supported. For more on Bloom
filter compression, see Michael Mitzenmacher's "Compressed Bloom
Filters",
https://www.eecs.harvard.edu/~michaelm/NEWWORK/postscripts/cbf2.pdf
  • Loading branch information
jbapple authored and rdblue committed Aug 13, 2019
1 parent fa53342 commit 556ebee
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 0 deletions.
10 changes: 10 additions & 0 deletions BloomFilter.md
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,14 @@ union BloomFilterHash {
1: XxHash XXHASH;
}
/**
* The compression used in the Bloom filter.
**/
struct Uncompressed {}
union BloomFilterCompression {
1: Uncompressed UNCOMPRESSED;
}
/**
* Bloom filter header is stored at beginning of Bloom filter data of each column
* and followed by its bitset.
Expand All @@ -274,6 +282,8 @@ struct BloomFilterPageHeader {
2: required BloomFilterAlgorithm algorithm;
/** The hash function used for Bloom filter. **/
3: required BloomFilterHash hash;
/** The compression used in the Bloom filter **/
4: required BloomFilterCompression compression;
}
struct ColumnMetaData {
Expand Down
11 changes: 11 additions & 0 deletions src/main/thrift/parquet.thrift
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,15 @@ union BloomFilterHash {
/** xxHash Strategy. **/
1: XxHash XXHASH;
}

/**
* The compression used in the Bloom filter.
**/
struct Uncompressed {}
union BloomFilterCompression {
1: Uncompressed UNCOMPRESSED;
}

/**
* Bloom filter header is stored at beginning of Bloom filter data of each column
* and followed by its bitset.
Expand All @@ -593,6 +602,8 @@ struct BloomFilterHeader {
2: required BloomFilterAlgorithm algorithm;
/** The hash function used for Bloom filter. **/
3: required BloomFilterHash hash;
/** The compression used in the Bloom filter **/
4: required BloomFilterCompression compression;
}

struct PageHeader {
Expand Down

0 comments on commit 556ebee

Please sign in to comment.