Skip to content

Support single-valued BigDecimal in schema, type conversion, SQL statements and minimum set of transforms. #8503

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jackie-Jiang merged 26 commits into from
Apr 30, 2022
Merged

Support single-valued BigDecimal in schema, type conversion, SQL statements and minimum set of transforms. #8503

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
8e4fce0
Support single-valued BigDecimal in schema and type conversion as wel…
nizarhejazi Apr 11, 2022
b673802
Merge branch 'master' into nhejazi-ingestion-bigdecimal
nizarhejazi Apr 11, 2022
e50ac81
fix checkstyle warnings
nizarhejazi Apr 11, 2022
f73f995
address comments
nizarhejazi Apr 13, 2022
96a8728
cleanup; address checkstyle warnings
nizarhejazi Apr 13, 2022
d96d401
cleanup a method
nizarhejazi Apr 13, 2022
f26632a
Merge branch 'master' into nhejazi-ingestion-bigdecimal
nizarhejazi Apr 13, 2022
1d738b4
add dictionary, raw inverted index reader etc for BIG_DECIMAL type
nizarhejazi Apr 17, 2022
98965b1
Merge branch 'master' into nhejazi-ingestion-bigdecimal
nizarhejazi Apr 17, 2022
8b5c0a5
address checkstyle warnings
nizarhejazi Apr 17, 2022
c578573
Not allow a dimension BIG_DECIMAL
nizarhejazi Apr 18, 2022
f8b8161
address comments
nizarhejazi Apr 24, 2022
1288ea2
Merge branch 'master' into nhejazi-ingestion-bigdecimal
nizarhejazi Apr 24, 2022
8266426
minor cleanup
nizarhejazi Apr 25, 2022
00d8389
address checkstyle warning
nizarhejazi Apr 25, 2022
b463cb5
Add getBigDecimalValue to all dicts. address comments
nizarhejazi Apr 27, 2022
a9452d8
Merge branch 'master' into nhejazi-ingestion-bigdecimal
nizarhejazi Apr 27, 2022
79a8312
address checkstyle warnings
nizarhejazi Apr 27, 2022
f5f6a7f
fix tests using JsonUtils
nizarhejazi Apr 27, 2022
b540714
address comments
nizarhejazi Apr 28, 2022
4f8d466
minor cleanup
nizarhejazi Apr 28, 2022
d4ef558
Not set withExactBigDecimals
nizarhejazi Apr 29, 2022
1f95428
Merge branch 'master' into nhejazi-ingestion-bigdecimal
nizarhejazi Apr 29, 2022
ab00896
update BigDecimalQueriesTest after merging master
nizarhejazi Apr 29, 2022
d4c0e8b
address comments, remove pinot-plugins.tar.gz
nizarhejazi Apr 29, 2022
149b476
address linter warning
nizarhejazi Apr 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/utils/DriverUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.pinot.client.utils;

import java.math.BigDecimal;
import java.net.URI;
import java.sql.Timestamp;
import java.sql.Types;
Expand Down Expand Up @@ -97,6 +98,9 @@ public static Integer getSQLDataType(String columnDataType) {
case "DOUBLE":
columnsSQLDataType = Types.DOUBLE;
break;
case "BIG_DECIMAL":
columnsSQLDataType = Types.DECIMAL;
break;
case "BOOLEAN":
columnsSQLDataType = Types.BOOLEAN;
break;
Expand Down Expand Up @@ -134,6 +138,9 @@ public static String getJavaClassName(String columnDataType) {
case "DOUBLE":
columnsJavaClassName = Double.class.getTypeName();
break;
case "BIG_DECIMAL":
columnsJavaClassName = BigDecimal.class.getTypeName();
break;
case "BOOLEAN":
columnsJavaClassName = Boolean.class.getTypeName();
break;
Expand Down
5 changes: 5 additions & 0 deletions pinot-common/src/main/java/org/apache/pinot/common/function/FunctionUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.pinot.common.function;

import java.math.BigDecimal;
import java.sql.Timestamp;
import java.util.HashMap;
import java.util.Map;
Expand All @@ -41,6 +42,7 @@ private FunctionUtils() {
put(Float.class, PinotDataType.FLOAT);
put(double.class, PinotDataType.DOUBLE);
put(Double.class, PinotDataType.DOUBLE);
put(BigDecimal.class, PinotDataType.BIG_DECIMAL);
put(boolean.class, PinotDataType.BOOLEAN);
put(Boolean.class, PinotDataType.BOOLEAN);
put(Timestamp.class, PinotDataType.TIMESTAMP);
Expand All @@ -64,6 +66,7 @@ private FunctionUtils() {
put(Long.class, PinotDataType.LONG);
put(Float.class, PinotDataType.FLOAT);
put(Double.class, PinotDataType.DOUBLE);
put(BigDecimal.class, PinotDataType.BIG_DECIMAL);
put(Timestamp.class, PinotDataType.TIMESTAMP);
put(String.class, PinotDataType.STRING);
put(byte[].class, PinotDataType.BYTES);
Expand All @@ -88,6 +91,7 @@ private FunctionUtils() {
put(Float.class, DataType.FLOAT);
put(double.class, DataType.DOUBLE);
put(Double.class, DataType.DOUBLE);
put(BigDecimal.class, DataType.BIG_DECIMAL);
put(boolean.class, DataType.BOOLEAN);
put(Boolean.class, DataType.BOOLEAN);
put(Timestamp.class, DataType.TIMESTAMP);
Expand All @@ -109,6 +113,7 @@ private FunctionUtils() {
put(Float.class, ColumnDataType.FLOAT);
put(double.class, ColumnDataType.DOUBLE);
put(Double.class, ColumnDataType.DOUBLE);
put(BigDecimal.class, ColumnDataType.BIG_DECIMAL);
put(boolean.class, ColumnDataType.BOOLEAN);
put(Boolean.class, ColumnDataType.BOOLEAN);
put(Timestamp.class, ColumnDataType.TIMESTAMP);
Expand Down
14 changes: 13 additions & 1 deletion pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.Serializable;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.util.Arrays;
Expand Down Expand Up @@ -242,6 +243,7 @@ public enum ColumnDataType {
LONG,
FLOAT,
DOUBLE,
BIG_DECIMAL,
BOOLEAN /* Stored as INT */,
TIMESTAMP /* Stored as LONG */,
STRING,
Expand All @@ -257,7 +259,7 @@ public enum ColumnDataType {
BYTES_ARRAY,
STRING_ARRAY;

private static final EnumSet<ColumnDataType> NUMERIC_TYPES = EnumSet.of(INT, LONG, FLOAT, DOUBLE);
private static final EnumSet<ColumnDataType> NUMERIC_TYPES = EnumSet.of(INT, LONG, FLOAT, DOUBLE, BIG_DECIMAL);
private static final EnumSet<ColumnDataType> INTEGRAL_TYPES = EnumSet.of(INT, LONG);
private static final EnumSet<ColumnDataType> ARRAY_TYPES = EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY,
DOUBLE_ARRAY, STRING_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY, BYTES_ARRAY);
Expand Down Expand Up @@ -316,6 +318,8 @@ public DataType toDataType() {
return DataType.FLOAT;
case DOUBLE:
return DataType.DOUBLE;
case BIG_DECIMAL:
return DataType.BIG_DECIMAL;
case BOOLEAN:
return DataType.BOOLEAN;
case TIMESTAMP:
Expand Down Expand Up @@ -345,6 +349,8 @@ public Serializable convert(Object value) {
return ((Number) value).floatValue();
case DOUBLE:
return ((Number) value).doubleValue();
case BIG_DECIMAL:
return (BigDecimal) value;
case BOOLEAN:
return (Integer) value == 1;
case TIMESTAMP:
Expand Down Expand Up @@ -380,6 +386,8 @@ public Serializable convert(Object value) {
*/
public Serializable format(Object value) {
switch (this) {
case BIG_DECIMAL:
return ((BigDecimal) value).toPlainString();
case TIMESTAMP:
assert value instanceof Timestamp;
return value.toString();
Expand All @@ -403,6 +411,8 @@ public Serializable convertAndFormat(Object value) {
return ((Number) value).floatValue();
case DOUBLE:
return ((Number) value).doubleValue();
case BIG_DECIMAL:
return (BigDecimal) value;
case BOOLEAN:
return (Integer) value == 1;
case TIMESTAMP:
Expand Down Expand Up @@ -514,6 +524,8 @@ public static ColumnDataType fromDataTypeSV(DataType dataType) {
return FLOAT;
case DOUBLE:
return DOUBLE;
case BIG_DECIMAL:
return BIG_DECIMAL;
case BOOLEAN:
return BOOLEAN;
case TIMESTAMP:
Expand Down
3 changes: 3 additions & 0 deletions pinot-common/src/main/java/org/apache/pinot/common/utils/DataTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.pinot.common.utils;

import java.io.IOException;
import java.math.BigDecimal;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -57,6 +58,8 @@ byte[] toBytes()

double getDouble(int rowId, int colId);

BigDecimal getBigDecimal(int rowId, int colId);

String getString(int rowId, int colId);

ByteArray getBytes(int rowId, int colId);
Expand Down
Loading