Skip to content

Commit

Permalink
Run spotless over iceberg-spark-3.3 and iceberg-api
Browse files Browse the repository at this point in the history
  • Loading branch information
@kbendick
kbendick committed Jul 28, 2022
1 parent 1b5a0e5 commit 24efcba
Show file tree
Hide file tree
Showing 6 changed files with 278 additions and 183 deletions.
40 changes: 20 additions & 20 deletions api/src/main/java/org/apache/iceberg/util/TruncateUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.util;

import java.math.BigDecimal;
Expand All @@ -25,22 +24,21 @@

/**
* Contains the logic for various {@code truncate} transformations for various types.
* <p>
* This utility class allows for the logic to be reused in different scenarios where input
* validation is done at different times either in org.apache.iceberg.transforms.Truncate
* and within defined SQL functions for different compute engines for usage in SQL.
* <p>
* In general, the inputs to the functions should have already been validated by the calling code,
* as different classes use truncate with different preprocessing. This generally means that
*
* <p>This utility class allows for the logic to be reused in different scenarios where input
* validation is done at different times either in org.apache.iceberg.transforms.Truncate and within
* defined SQL functions for different compute engines for usage in SQL.
*
* <p>In general, the inputs to the functions should have already been validated by the calling
* code, as different classes use truncate with different preprocessing. This generally means that
* the truncation width is positive and the value to truncate is non-null.
* <p>
* See also {@linkplain UnicodeUtil#truncateString(CharSequence, int)} and
* {@link BinaryUtil#truncateBinary(ByteBuffer, int)}
*
* <p>See also {@linkplain UnicodeUtil#truncateString(CharSequence, int)} and {@link
* BinaryUtil#truncateBinary(ByteBuffer, int)}
*/
public class TruncateUtil {
// not meant to be instantiated
private TruncateUtil() {
}

private TruncateUtil() {}

public static ByteBuffer truncateByteBuffer(int width, ByteBuffer value) {
ByteBuffer ret = value.duplicate();
Expand All @@ -65,12 +63,14 @@ public static long truncateLong(int width, long value) {
}

public static BigDecimal truncateDecimal(BigInteger unscaledWidth, BigDecimal value) {
BigDecimal remainder = new BigDecimal(
value.unscaledValue()
.remainder(unscaledWidth)
.add(unscaledWidth)
.remainder(unscaledWidth),
value.scale());
BigDecimal remainder =
new BigDecimal(
value
.unscaledValue()
.remainder(unscaledWidth)
.add(unscaledWidth)
.remainder(unscaledWidth),
value.scale());

return value.subtract(remainder);
}
Expand Down
17 changes: 12 additions & 5 deletions spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/BaseCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,11 @@
import org.apache.spark.sql.connector.iceberg.catalog.ProcedureCatalog;

abstract class BaseCatalog
implements StagingTableCatalog, ProcedureCatalog, SupportsNamespaces, HasIcebergCatalog, FunctionCatalog {
implements StagingTableCatalog,
ProcedureCatalog,
SupportsNamespaces,
HasIcebergCatalog,
FunctionCatalog {

@Override
public Procedure loadProcedure(Identifier ident) throws NoSuchProcedureException {
Expand All @@ -55,7 +59,8 @@ public Procedure loadProcedure(Identifier ident) throws NoSuchProcedureException

@Override
public Identifier[] listFunctions(String[] namespace) throws NoSuchNamespaceException {
if (namespace.length == 0 || (namespace.length == 1 && namespace[0].equalsIgnoreCase("system"))) {
if (namespace.length == 0
|| (namespace.length == 1 && namespace[0].equalsIgnoreCase("system"))) {
return SparkFunctions.list().stream()
.map(name -> Identifier.of(namespace, name))
.toArray(Identifier[]::new);
Expand All @@ -71,9 +76,11 @@ public UnboundFunction loadFunction(Identifier ident) throws NoSuchFunctionExcep
String[] namespace = ident.namespace();
String name = ident.name();

// Allow for empty namespace as Spark's storage partitioned joins look up the corresponding transform
// functions, like `bucket`, with an empty namespace
if (namespace.length == 0 || (namespace.length == 1 && namespace[0].equalsIgnoreCase("system"))) {
// Allow for empty namespace as Spark's storage partitioned joins look up
// the corresponding functions to generate transforms for partitioning
// with an empty namespace, such as `bucket`.
if (namespace.length == 0
|| (namespace.length == 1 && namespace[0].equalsIgnoreCase("system"))) {
UnboundFunction func = SparkFunctions.load(name);
if (func != null) {
return func;
Expand Down
14 changes: 5 additions & 9 deletions spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/functions/SparkFunctions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.spark.functions;

import java.util.List;
Expand All @@ -28,17 +27,15 @@

public class SparkFunctions {

private SparkFunctions() {
}
private SparkFunctions() {}

private static final Map<String, UnboundFunction> FUNCTIONS = ImmutableMap.of(
"truncate", new TruncateFunction()
);
private static final Map<String, UnboundFunction> FUNCTIONS =
ImmutableMap.of("truncate", new TruncateFunction());

private static final List<String> FUNCTION_NAMES = ImmutableList.copyOf(FUNCTIONS.keySet());

// Functions that are added to all Iceberg catalogs get registered under the `system` namespace, so a list of
// names alone is returned.
// Functions that are added to all Iceberg catalogs should be accessed with either the `system`
// namespace or no namespace at all, so a list of names alone is returned.
public static List<String> list() {
return FUNCTION_NAMES;
}
Expand All @@ -48,5 +45,4 @@ public static UnboundFunction load(String name) {
UnboundFunction func = FUNCTIONS.get(name.toLowerCase(Locale.ROOT));
return func;
}

}
81 changes: 47 additions & 34 deletions spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/functions/TruncateFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.spark.functions;

import java.math.BigInteger;
Expand Down Expand Up @@ -49,44 +48,55 @@
import org.apache.spark.unsafe.types.UTF8String;

/**
* Implementation of {@link UnboundFunction} that matches the <b>truncate</b> transformation.
* This unbound function is registered with the {@link org.apache.iceberg.spark.SparkCatalog}
* such that the function can be used as {@code truncate(width, col)} or {@code truncate(2, col)}.
* <p>
* Specific {@link BoundFunction} implementations are resolved based on their input types. As with transforms, the
* truncation width must be non-negative.
* <p>
* For efficiency in generated code, the {@code width} is not validated.
* <b>It is the responsibility of calling code of these functions to not call truncate with a non-positive width.</b>
* Implementation of {@link UnboundFunction} that matches the <b>truncate</b> transformation. This
* unbound function is registered with the {@link org.apache.iceberg.spark.SparkCatalog} such that
* the function can be used as {@code truncate(width, col)} or {@code truncate(2, col)}.
*
* <p>Specific {@link BoundFunction} implementations are resolved based on their input types. As
* with transforms, the truncation width must be non-negative.
*
* <p>For efficiency in generated code, the {@code width} is not validated. <b>It is the
* responsibility of calling code of these functions to not call truncate with a non-positive
* width.</b>
*/
public class TruncateFunction implements UnboundFunction {
private static final List<DataType> truncateableAtomicTypes = ImmutableList.of(
DataTypes.ByteType, DataTypes.ShortType, DataTypes.IntegerType, DataTypes.LongType,
DataTypes.StringType, DataTypes.BinaryType);
private static final List<DataType> truncateableAtomicTypes =
ImmutableList.of(
DataTypes.ByteType,
DataTypes.ShortType,
DataTypes.IntegerType,
DataTypes.LongType,
DataTypes.StringType,
DataTypes.BinaryType);

private static void validateTruncationFieldType(DataType dt) {
if (truncateableAtomicTypes.stream().noneMatch(type -> type.sameType(dt)) &&
!(dt instanceof DecimalType)) {
String expectedTypes = "[ByteType, ShortType, IntegerType, LongType, StringType, BinaryType, DecimalType]";
if (truncateableAtomicTypes.stream().noneMatch(type -> type.sameType(dt))
&& !(dt instanceof DecimalType)) {
String expectedTypes =
"[ByteType, ShortType, IntegerType, LongType, StringType, BinaryType, DecimalType]";
throw new UnsupportedOperationException(
String.format("Invalid input type to truncate. Expected one of %s, but found %s", expectedTypes, dt));
String.format(
"Invalid input type to truncate. Expected one of %s, but found %s",
expectedTypes, dt));
}
}

private static void validateTruncationWidthType(DataType widthType) {
if (!DataTypes.IntegerType.sameType(widthType) &&
!DataTypes.ShortType.sameType(widthType) &&
!DataTypes.ByteType.sameType(widthType)) {
if (!DataTypes.IntegerType.sameType(widthType)
&& !DataTypes.ShortType.sameType(widthType)
&& !DataTypes.ByteType.sameType(widthType)) {
throw new UnsupportedOperationException(
"Expected truncation width to be one of [ByteType, ShortType, IntegerType], but found " + widthType);
"Expected truncation width to be one of [ByteType, ShortType, IntegerType], but found "
+ widthType);
}
}

@Override
public BoundFunction bind(StructType inputType) {
if (inputType.fields().length != 2) {
throw new UnsupportedOperationException(
String.format("Invalid input type. Expected 2 fields but found %s", inputType.fields().length));
String.format(
"Invalid input type. Expected 2 fields but found %s", inputType.fields().length));
}

StructField widthField = inputType.apply(0);
Expand All @@ -108,10 +118,9 @@ public BoundFunction bind(StructType inputType) {
return new TruncateDecimal(
((DecimalType) toTruncateDataType).precision(),
((DecimalType) toTruncateDataType).scale());
} else if (
toTruncateDataType instanceof StringType ||
toTruncateDataType instanceof VarcharType ||
toTruncateDataType instanceof CharType) {
} else if (toTruncateDataType instanceof StringType
|| toTruncateDataType instanceof VarcharType
|| toTruncateDataType instanceof CharType) {
return new TruncateString();
} else if (toTruncateDataType instanceof BinaryType) {
return new TruncateBinary();
Expand All @@ -122,8 +131,8 @@ public BoundFunction bind(StructType inputType) {

@Override
public String description() {
return "Truncate - The Iceberg truncate function used for truncate partition transformations.\n" +
"\tCalled with the truncation width as the first argument: e.g. system.truncate(width, col)";
return "Truncate - The Iceberg truncate function used for truncate partition transformations.\n"
+ "\tCalled with the truncation width as the first argument: e.g. system.truncate(width, col)";
}

@Override
Expand All @@ -145,7 +154,7 @@ public static byte invoke(int width, byte value) {

@Override
public DataType[] inputTypes() {
return new DataType[]{DataTypes.IntegerType, DataTypes.ByteType};
return new DataType[] {DataTypes.IntegerType, DataTypes.ByteType};
}

@Override
Expand Down Expand Up @@ -292,7 +301,7 @@ public String produceResult(InternalRow input) {
Integer width = readAndValidateWidth(input);

UTF8String toTruncate = !input.isNullAt(1) ? input.getUTF8String(1) : null;
UTF8String result = toTruncate != null ? invoke(width, toTruncate) : null;
UTF8String result = toTruncate != null ? invoke(width, toTruncate) : null;
return result != null ? result.toString() : null;
}
}
Expand All @@ -304,7 +313,8 @@ public static byte[] invoke(int width, byte[] value) {
return null;
}

return ByteBuffers.toByteArray(TruncateUtil.truncateByteBuffer(width, ByteBuffer.wrap(value)));
return ByteBuffers.toByteArray(
TruncateUtil.truncateByteBuffer(width, ByteBuffer.wrap(value)));
}

@Override
Expand Down Expand Up @@ -346,7 +356,8 @@ public static Decimal invoke(int width, Decimal value) {
return null;
}

return Decimal.apply(TruncateUtil.truncateDecimal(BigInteger.valueOf(width), value.toJavaBigDecimal()));
return Decimal.apply(
TruncateUtil.truncateDecimal(BigInteger.valueOf(width), value.toJavaBigDecimal()));
}

@Override
Expand All @@ -361,7 +372,8 @@ public DataType resultType() {

@Override
public String canonicalName() {
return String.format("org.apache.iceberg.spark.functions.truncate[width](decimal(%d,%d))", precision, scale);
return String.format(
"org.apache.iceberg.spark.functions.truncate[width](decimal(%d,%d))", precision, scale);
}

@Override
Expand All @@ -380,7 +392,8 @@ private static Integer readAndValidateWidth(InternalRow input) {
}

if (width <= 0) {
throw new IllegalArgumentException(String.format("Invalid truncate width: %s (must be > 0)", width));
throw new IllegalArgumentException(
String.format("Invalid truncate width: %s (must be > 0)", width));
}

return width;
Expand Down
Loading

0 comments on commit 24efcba

Please sign in to comment.