Merge 145a1dc into 4bcc7a6

Author: tustvold

parquet/src/encodings/decoding.rs

@@ -433,7 +433,6 @@ pub struct DeltaBitPackDecoder<T: DataType> {
     initialized: bool,
 
     // Header info
-
     /// The number of values in each block
     block_size: usize,
     /// The number of values that remain to be read in the current page
@@ -444,7 +443,6 @@ pub struct DeltaBitPackDecoder<T: DataType> {
     values_per_mini_block: usize,
 
     // Per block info
-
     /// The minimum delta in the block
     min_delta: T::T,
     /// The byte offset of the end of the current block
@@ -639,6 +637,7 @@ where
             self.last_value = value;
             buffer[0] = value;
             read += 1;
+            self.values_left -= 1;
         }
 
         while read != to_read {
@@ -668,9 +667,9 @@ where
 
             read += batch_read;
             self.mini_block_remaining -= batch_read;
+            self.values_left -= batch_read;
         }
 
-        self.values_left -= to_read;
         Ok(to_read)
     }
 

parquet/src/encodings/encoding.rs

@@ -565,7 +565,7 @@ impl<T: DataType> DeltaBitPackEncoder<T> {
             return Ok(());
         }
 
-        let mut min_delta = i64::max_value();
+        let mut min_delta = i64::MAX;
         for i in 0..self.values_in_block {
             min_delta = cmp::min(min_delta, self.deltas[i]);
         }
@@ -581,6 +581,13 @@ impl<T: DataType> DeltaBitPackEncoder<T> {
             // values left
             let n = cmp::min(self.mini_block_size, self.values_in_block);
             if n == 0 {
+                // Decoders should be agnostic to the padding value, we therefore use 0xFF
+                // when running tests. However, not all implementations may handle this correctly
+                // so pad with 0 when not running tests
+                let pad_value = cfg!(test).then(|| 0xFF).unwrap_or(0);
+                for j in i..self.num_mini_blocks {
+                    self.bit_writer.write_at(offset + j, pad_value);
+                }
                 break;
             }
 
@@ -610,8 +617,8 @@ impl<T: DataType> DeltaBitPackEncoder<T> {
             self.values_in_block -= n;
         }
 
-        assert!(
-            self.values_in_block == 0,
+        assert_eq!(
+            self.values_in_block, 0,
             "Expected 0 values in block, found {}",
             self.values_in_block
         );