Add data_page_version & size_statistics_level to WriterProperties

cpp/Enums.cpp

@@ -110,5 +110,12 @@ namespace
 		static_assert((int) ::arrow::Type::type::LIST == 25);
 		static_assert((int) ::arrow::Type::type::LARGE_LIST == 36);
 		static_assert((int) ::arrow::Type::type::LIST_VIEW == 41);
+
+		static_assert((int) ::ParquetDataPageVersion::V1 == 0);
+		static_assert((int) ::ParquetDataPageVersion::V2 == 1);
+
+		static_assert((int) ::parquet::SizeStatisticsLevel::None == 0);
+		static_assert((int) ::parquet::SizeStatisticsLevel::ColumnChunk == 1);
+		static_assert((int) ::parquet::SizeStatisticsLevel::PageAndColumnChunk == 2);
 	}
 }

cpp/WriterProperties.cpp

@@ -167,4 +167,14 @@ extern "C"
 	{
 		TRYCATCH(*store_decimal_as_integer = (*writer_properties)->store_decimal_as_integer();)
 	}
+
+	PARQUETSHARP_EXPORT ExceptionInfo* WriterProperties_Data_Page_Version(const std::shared_ptr<WriterProperties>* writer_properties, ::ParquetDataPageVersion* data_page_version)
+	{
+		TRYCATCH(*data_page_version = (*writer_properties)->data_page_version();)
+	}
+
+	PARQUETSHARP_EXPORT ExceptionInfo* WriterProperties_Size_Statistics_Level(const std::shared_ptr<WriterProperties>* writer_properties, ::SizeStatisticsLevel* size_statistics_level)
+	{
+		TRYCATCH(*size_statistics_level = (*writer_properties)->size_statistics_level();)
+	}
 }

cpp/WriterPropertiesBuilder.cpp

@@ -246,4 +246,14 @@ extern "C"
 	{
 		TRYCATCH(builder->max_statistics_size(max_statistics_size);)
 	}
+
+	PARQUETSHARP_EXPORT ExceptionInfo* WriterPropertiesBuilder_Data_Page_Version(WriterProperties::Builder* builder, ::ParquetDataPageVersion data_page_version)
+	{
+		TRYCATCH(builder->data_page_version(data_page_version);)
+	}
+
+	PARQUETSHARP_EXPORT ExceptionInfo* WriterPropertiesBuilder_Set_Size_Statistics_Level(WriterProperties::Builder* builder, ::SizeStatisticsLevel size_statistics_level)
+	{
+		TRYCATCH(builder->set_size_statistics_level(size_statistics_level);)
+	}
 }

csharp.test/TestParquetDataPageVersion.cs

@@ -0,0 +1,22 @@
+using NUnit.Framework;
+
+namespace ParquetSharp.Test
+{
+    [TestFixture]
+    public class TestParquetDataPageVersion
+    {
+        /// <summary>
+        /// We pass custom enum ParquetSharp.ParquetDataPageVersion values from the Arrow C# library into the Arrow C++ library
+        /// and expect that the integer representations are the same so that no conversion is required.
+        /// Verify this is true.
+        ///
+        /// See Enums.cpp in the C++ library which uses static assertions to verify the same conditions on the C++ side.
+        /// </summary>
+        [Test]
+        public void VerifyParquetDataPageVersionEnumValues()
+        {
+            Assert.That((int) ParquetDataPageVersion.V1, Is.EqualTo(0));
+            Assert.That((int) ParquetDataPageVersion.V2, Is.EqualTo(1));
+        }
+    }
+}

csharp.test/TestSizeStatisticsLevel.cs

@@ -0,0 +1,23 @@
+using NUnit.Framework;
+
+namespace ParquetSharp.Test
+{
+    [TestFixture]
+    public class TestSizeStatisticsLevel
+    {
+        /// <summary>
+        /// We pass custom enum ParquetSharp.SizeStatisticsLevel values from the Arrow C# library into the Arrow C++ library
+        /// and expect that the integer representations are the same so that no conversion is required.
+        /// Verify this is true.
+        ///
+        /// See Enums.cpp in the C++ library which uses static assertions to verify the same conditions on the C++ side.
+        /// </summary>
+        [Test]
+        public void VerifySizeStatisticsLevelEnumValues()
+        {
+            Assert.That((int) SizeStatisticsLevel.None, Is.EqualTo(0));
+            Assert.That((int) SizeStatisticsLevel.ColumnChunk, Is.EqualTo(1));
+            Assert.That((int) SizeStatisticsLevel.PageAndColumnChunk, Is.EqualTo(2));
+        }
+    }
+}

csharp.test/TestWriterProperties.cs

@@ -29,6 +29,9 @@ public static void TestDefaultProperties()
             Assert.That(p.MemoryPool.BackendName, Is.Not.Empty);
             Assert.False(p.StoreDecimalAsInteger);
             Assert.AreEqual(4096, p.MaxStatisticsSize(new ColumnPath("anypath")));
+            Assert.AreEqual(p.DataPageVersion, ParquetDataPageVersion.V1);
+            Assert.AreEqual(p.SizeStatisticsLevel, SizeStatisticsLevel.PageAndColumnChunk);
+            Assert.AreEqual(4096, p.MaxStatisticsSize(new ColumnPath("anypath")));
         }
 
         [Test]
@@ -50,6 +53,8 @@ public static void TestPropertiesBuilder()
                 .MemoryPool(MemoryPool.SystemMemoryPool())
                 .EnableStoreDecimalAsInteger()
                 .SetMaxStatisticsSize(512)
+                .DataPageVersion(ParquetDataPageVersion.V2)
+                .SetSizeStatisticsLevel(SizeStatisticsLevel.ColumnChunk)
                 .Build();
 
             Assert.AreEqual("Meeeee!!!", p.CreatedBy);
@@ -67,6 +72,8 @@ public static void TestPropertiesBuilder()
             Assert.AreEqual("system", p.MemoryPool.BackendName);
             Assert.True(p.StoreDecimalAsInteger);
             Assert.AreEqual(512, p.MaxStatisticsSize(new ColumnPath("anypath")));
+            Assert.AreEqual(p.DataPageVersion, ParquetDataPageVersion.V2);
+            Assert.AreEqual(p.SizeStatisticsLevel, SizeStatisticsLevel.ColumnChunk);
         }
 
         [Test]

csharp/ParquetDataPageVersion.cs

@@ -0,0 +1,8 @@
+namespace ParquetSharp
+{
+    public enum ParquetDataPageVersion
+    {
+        V1 = 0,
+        V2 = 1
+    }
+}

csharp/PublicAPI.Unshipped.txt

@@ -14,4 +14,15 @@ ParquetSharp.ReaderProperties.SetFooterReadSize(long size) -> void
 ParquetSharp.WriterProperties.StoreDecimalAsInteger.get -> bool
 ParquetSharp.WriterPropertiesBuilder.EnableStoreDecimalAsInteger() -> ParquetSharp.WriterPropertiesBuilder!
 ParquetSharp.WriterPropertiesBuilder.DisableStoreDecimalAsInteger() -> ParquetSharp.WriterPropertiesBuilder!
-ParquetSharp.WriterPropertiesBuilder.SetMaxStatisticsSize(ulong maxStatisticsSize) -> ParquetSharp.WriterPropertiesBuilder!
+ParquetSharp.WriterPropertiesBuilder.SetMaxStatisticsSize(ulong maxStatisticsSize) -> ParquetSharp.WriterPropertiesBuilder!
+ParquetSharp.ParquetDataPageVersion
+ParquetSharp.ParquetDataPageVersion.V1 = 0 -> ParquetSharp.ParquetDataPageVersion
+ParquetSharp.ParquetDataPageVersion.V2 = 1 -> ParquetSharp.ParquetDataPageVersion
+ParquetSharp.WriterProperties.DataPageVersion.get -> ParquetSharp.ParquetDataPageVersion
+ParquetSharp.WriterPropertiesBuilder.DataPageVersion(ParquetSharp.ParquetDataPageVersion dataPageVersion) -> ParquetSharp.WriterPropertiesBuilder!
+ParquetSharp.SizeStatisticsLevel
+ParquetSharp.SizeStatisticsLevel.None = 0 -> ParquetSharp.SizeStatisticsLevel
+ParquetSharp.SizeStatisticsLevel.ColumnChunk = 1 -> ParquetSharp.SizeStatisticsLevel
+ParquetSharp.SizeStatisticsLevel.PageAndColumnChunk = 2 -> ParquetSharp.SizeStatisticsLevel
+ParquetSharp.WriterProperties.SizeStatisticsLevel.get -> ParquetSharp.SizeStatisticsLevel
+ParquetSharp.WriterPropertiesBuilder.SetSizeStatisticsLevel(ParquetSharp.SizeStatisticsLevel level) -> ParquetSharp.WriterPropertiesBuilder!

csharp/SizeStatisticsLevel.cs

@@ -0,0 +1,9 @@
+namespace ParquetSharp
+{
+    public enum SizeStatisticsLevel : byte
+    {
+        None = 0,
+        ColumnChunk = 1,
+        PageAndColumnChunk = 2
+    }
+}

csharp/WriterProperties.cs

@@ -269,6 +269,18 @@ public SortingColumn[] SortingColumns()
         /// </summary>
         public bool StoreDecimalAsInteger => ExceptionInfo.Return<bool>(Handle, WriterProperties_Store_Decimal_As_Integer);
 
+        /// <summary>
+        /// The data page version to use when writing data pages.
+        /// Default is V1.
+        /// </summary>
+        public ParquetDataPageVersion DataPageVersion => ExceptionInfo.Return<ParquetDataPageVersion>(Handle, WriterProperties_Data_Page_Version);
+
+        /// <summary>
+        /// The level to write size statistics for all columns. 
+        /// Default is None.
+        /// </summary>
+        public SizeStatisticsLevel SizeStatisticsLevel => ExceptionInfo.Return<SizeStatisticsLevel>(Handle, WriterProperties_Size_Statistics_Level);
+
         internal readonly ParquetHandle Handle;
 
         [DllImport(ParquetDll.Name)]
@@ -348,5 +360,11 @@ public SortingColumn[] SortingColumns()
 
         [DllImport(ParquetDll.Name)]
         private static extern IntPtr WriterProperties_Store_Decimal_As_Integer(IntPtr writerProperties, out bool storeDecimalAsInteger);
+
+        [DllImport(ParquetDll.Name)]
+        private static extern IntPtr WriterProperties_Data_Page_Version(IntPtr writerProperties, out ParquetDataPageVersion dataPageVersion);
+
+        [DllImport(ParquetDll.Name)]
+        private static extern IntPtr WriterProperties_Size_Statistics_Level(IntPtr writerProperties, out SizeStatisticsLevel sizeStatisticsLevel);
     }
 }

csharp/WriterPropertiesBuilder.cs

@@ -662,6 +662,32 @@ public WriterPropertiesBuilder DisableStoreDecimalAsInteger()
             return this;
         }
 
+        /// <summary>
+        /// Specify the data page version.
+        /// </summary>
+        public WriterPropertiesBuilder DataPageVersion(ParquetDataPageVersion dataPageVersion)
+        {
+            ExceptionInfo.Check(WriterPropertiesBuilder_Data_Page_Version(_handle.IntPtr, dataPageVersion));
+            GC.KeepAlive(_handle);
+            return this;
+        }
+
+        /// <summary>
+        /// Set the level to write size statistics for all columns. 
+        /// 
+        /// Note that if page index is not enabled, 
+        /// page level size statistics will not be written even if the level
+        /// is set to PageAndColumnChunk.
+        /// </summary>
+        /// <param name="level"> The level to write size statistics</param>
+        /// <returns>This builder instance.</returns>
+        public WriterPropertiesBuilder SetSizeStatisticsLevel(SizeStatisticsLevel level)
+        {
+            ExceptionInfo.Check(WriterPropertiesBuilder_Set_Size_Statistics_Level(_handle.IntPtr, level));
+            GC.KeepAlive(_handle);
+            return this;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static void OnDefaultProperty<T>(T? defaultPropertyValue, Action<T> setProperty)
             where T : struct
@@ -819,6 +845,13 @@ private static void OnDefaultRefProperty<T>(T? defaultPropertyValue, Action<T> s
         [DllImport(ParquetDll.Name)]
         private static extern IntPtr WriterPropertiesBuilder_Set_Max_Statistics_Size(IntPtr builder, ulong maxStatisticsSize);
 
+        [DllImport(ParquetDll.Name)]
+        private static extern IntPtr WriterPropertiesBuilder_Data_Page_Version(IntPtr builder, ParquetDataPageVersion dataPageVersion);
+
+        [DllImport(ParquetDll.Name)]
+        private static extern IntPtr WriterPropertiesBuilder_Set_Size_Statistics_Level(IntPtr builder, SizeStatisticsLevel level);
+
+
         private readonly ParquetHandle _handle;
     }
 }