Feature compatible sr bitmap

paimon-ext/lib/org.apache.paimon.factories.Factory

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.paimon.catalog.FileSystemCatalogFactory
+org.apache.paimon.jdbc.JdbcCatalogFactory
+org.apache.paimon.jdbc.JdbcCatalogLockFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldBoolAndAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldBoolOrAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldCollectAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldFirstNonNullValueAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldFirstNonNullValueAggLegacyFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldFirstValueAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldHllSketchAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldLastNonNullValueAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldLastValueAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldListaggAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldMaxAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldMergeMapAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldMinAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldNestedUpdateAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldPrimaryKeyAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldProductAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldRoaringBitmap32AggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldRoaringBitmap64AggFactory
+org.apache.paimon.mergetree.compact.aggregate.ext.factory.FieldStarRocksBitmapAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldSumAggFactory
+org.apache.paimon.mergetree.compact.aggregate.factory.FieldThetaSketchAggFactory
+org.apache.paimon.rest.RESTCatalogFactory
+org.apache.paimon.rest.auth.BearTokenCredentialsProviderFactory
+org.apache.paimon.rest.auth.BearTokenFileCredentialsProviderFactory

paimon-ext/pom.xml

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.paimon</groupId>
+        <artifactId>paimon-parent</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>paimon-ext</artifactId>
+    <name>Paimon : Extensions</name>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.starrocks</groupId>
+            <artifactId>hive-bitmap-udf</artifactId>
+            <version>1.0.0</version>
+            <scope>provided</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

paimon-ext/readme.md

@@ -0,0 +1,22 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+
+- 首先执行安装jar依赖到本地maven repo
+```shell
+cd paimon-ext/lib
+mvn install:install-file -Dfile=hive-bitmap-udf-1.0.0.jar -DgroupId=com.starrocks -DartifactId=hive-bitmap-udf -Dversion=1.0.0 -Dpackaging=jar
+```

paimon-ext/src/main/java/org/apache/paimon/mergetree/compact/aggregate/ext/FieldStarRocksBitmapAgg.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.mergetree.compact.aggregate.ext;
+
+import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator;
+import org.apache.paimon.types.VarBinaryType;
+
+import com.starrocks.types.BitmapValue;
+
+import java.io.IOException;
+
+/**
+ * roaring bitmap aggregate a field of a row. The current class is designed to be compatible with
+ * the bitmap format of StarRocks.
+ */
+public class FieldStarRocksBitmapAgg extends FieldAggregator {
+
+    private static final long serialVersionUID = 1L;
+    private BitmapValue acc;
+    private BitmapValue input;
+
+    public FieldStarRocksBitmapAgg(String name, VarBinaryType dataType) {
+        super(name, dataType);
+    }
+
+    @Override
+    public Object agg(Object accumulator, Object inputField) {
+        if (accumulator == null || inputField == null) {
+            return accumulator == null ? inputField : accumulator;
+        }
+
+        try {
+            this.acc = BitmapValue.bitmapFromBytes((byte[]) accumulator);
+            this.input = BitmapValue.bitmapFromBytes((byte[]) inputField);
+            this.acc.or(this.input);
+            return BitmapValue.bitmapToBytes(this.acc);
+        } catch (IOException e) {
+            throw new RuntimeException("Unable to se/deserialize roaring bitmap.", e);
+        } finally {
+            if (this.acc != null) {
+                this.acc.clear();
+            }
+            if (this.input != null) {
+                this.input.clear();
+            }
+        }
+    }
+}

paimon-ext/src/main/java/org/apache/paimon/mergetree/compact/aggregate/ext/factory/FieldStarRocksBitmapAggFactory.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.mergetree.compact.aggregate.ext.factory;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.mergetree.compact.aggregate.ext.FieldStarRocksBitmapAgg;
+import org.apache.paimon.mergetree.compact.aggregate.factory.FieldAggregatorFactory;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.VarBinaryType;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** Factory for #{@link FieldStarRocksBitmapAgg}. */
+public class FieldStarRocksBitmapAggFactory implements FieldAggregatorFactory {
+
+    public static final String NAME = "bitmap_agg";
+
+    @Override
+    public FieldStarRocksBitmapAgg create(DataType fieldType, CoreOptions options, String field) {
+        checkArgument(
+                fieldType instanceof VarBinaryType,
+                "Data type for roaring bitmap column must be 'VarBinaryType' but was '%s'.",
+                fieldType);
+        return new FieldStarRocksBitmapAgg(identifier(), (VarBinaryType) fieldType);
+    }
+
+    @Override
+    public String identifier() {
+        return NAME;
+    }
+}

paimon-ext/src/main/resources/META-INF/NOTICE

@@ -0,0 +1,8 @@
+paimon-core
+Copyright 2023-2024 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
+- com.squareup.okhttp3:okhttp:4.12.0

paimon-ext/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.paimon.mergetree.compact.aggregate.ext.factory.FieldStarRocksBitmapAggFactory

paimon-ext/src/test/java/org/apache/paimon/mergetree/compact/aggregate/ext/FieldAggregatorTest.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.mergetree.compact.aggregate.ext;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator;
+import org.apache.paimon.mergetree.compact.aggregate.ext.factory.FieldStarRocksBitmapAggFactory;
+import org.apache.paimon.mergetree.compact.aggregate.factory.FieldAggregatorFactory;
+import org.apache.paimon.types.DataTypes;
+
+import com.starrocks.types.BitmapValue;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** test whether {@link FieldAggregator}' subclasses behaviors are expected. */
+public class FieldAggregatorTest {
+
+    @Test
+    public void testFieldStarRocksBitmapAgg() throws IOException {
+        FieldStarRocksBitmapAgg agg =
+                new FieldStarRocksBitmapAggFactory().create(DataTypes.VARBINARY(20), null, null);
+        testAgg(agg);
+    }
+
+    @Test
+    public void testCustomAgg() throws IOException {
+        FieldAggregator agg =
+                FieldAggregatorFactory.create(
+                        DataTypes.VARBINARY(20),
+                        "bitmap",
+                        "bitmap_agg",
+                        CoreOptions.fromMap(new HashMap<>()));
+
+        testAgg(agg);
+    }
+
+    private void testAgg(FieldAggregator agg) throws IOException {
+        byte[] inputVal = BitmapValue.bitmapToBytes(new BitmapValue(1L));
+        BitmapValue bm1 = new BitmapValue(2L);
+        bm1.add(3L);
+        byte[] acc1 = BitmapValue.bitmapToBytes(bm1);
+
+        BitmapValue bm2 = new BitmapValue(1L);
+        bm2.add(2L);
+        bm2.add(3L);
+        byte[] acc2 = BitmapValue.bitmapToBytes(bm2);
+
+        assertThat(agg.agg(null, null)).isNull();
+
+        byte[] result1 = (byte[]) agg.agg(null, inputVal);
+        assertThat(inputVal).isEqualTo(result1);
+
+        byte[] result2 = (byte[]) agg.agg(acc1, null);
+        assertThat(result2).isEqualTo(acc1);
+
+        byte[] result3 = (byte[]) agg.agg(acc1, inputVal);
+        assertThat(result3).isEqualTo(acc2);
+
+        byte[] result4 = (byte[]) agg.agg(acc2, inputVal);
+        assertThat(result4).isEqualTo(acc2);
+
+        assertThat(BitmapValue.bitmapFromBytes(result4).cardinality()).isEqualTo(3L);
+    }
+}

pom.xml

@@ -70,6 +70,7 @@ under the License.
         <module>paimon-arrow</module>
         <module>tools/ci/paimon-ci-tools</module>
         <module>paimon-open-api</module>
+        <module>paimon-ext</module>
     </modules>
 
     <properties>