Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Support for user-defined enums #14691

Closed
wants to merge 13 commits into from
Closed
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,18 @@
*/
package com.facebook.presto.cli;

import com.facebook.presto.client.Column;
import com.facebook.presto.client.StatementClient;
import com.facebook.presto.common.type.TypeMetadata;
import com.google.common.collect.ImmutableList;
import io.airlift.units.Duration;

import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;

import static io.airlift.units.Duration.nanosSince;
Expand All @@ -44,19 +49,46 @@ public OutputHandler(OutputPrinter printer)
this.printer = requireNonNull(printer, "printer is null");
}

public void processRow(List<?> row)
public void processRow(List<?> row, List<Column> columns)
throws IOException
{
if (rowBuffer.isEmpty()) {
bufferStart = System.nanoTime();
}

rowBuffer.add(row);
rowBuffer.add(prettyPrintRow(row, columns));
if (rowBuffer.size() >= MAX_BUFFERED_ROWS) {
flush(false);
}
}

private List<?> prettyPrintRow(List<?> row, List<Column> columns)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

actually this doesn't work with complex types (row, map, array...)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rebuild switch-case logic here

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I might just call this function processRow or transform.

{
ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
for (int i = 0; i < row.size(); i++) {
Object value = row.get(i);
Column column = columns.get(i);
TypeMetadata typeInfo = column.getTypeMetadata();
if (typeInfo == null || typeInfo.getEnumValues() == null) {
builder.add(value);
continue;
}
builder.add(getDisplayValueForEnum(value, typeInfo.getEnumValues(), column.getType()));
}
return builder.build();
}

private Object getDisplayValueForEnum(Object value, Map<String, String> enumEntries, String enumName)
{
Optional<String> key = enumEntries.entrySet().stream()
.filter(e -> e.getValue().equals(value.toString()))
.map(Map.Entry::getKey).findFirst();
if (key.isPresent()) {
return String.format("%s'%s'", enumName, key.get());
}
return value;
}

@Override
public void close()
throws IOException
Expand All @@ -74,7 +106,7 @@ public void processRows(StatementClient client)
Iterable<List<Object>> data = client.currentData().getData();
if (data != null) {
for (List<Object> row : data) {
processRow(unmodifiableList(row));
processRow(unmodifiableList(row), client.currentData().getColumns());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package com.facebook.presto.client;

import com.facebook.presto.common.type.Type;
import com.facebook.presto.common.type.TypeMetadata;
import com.facebook.presto.common.type.TypeSignature;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
Expand All @@ -28,26 +29,33 @@ public class Column
private final String name;
private final String type;
private final ClientTypeSignature typeSignature;
private final TypeMetadata typeMetadata;

public Column(String name, Type type)
{
this(name, type.getTypeSignature());
this(
name,
type.getTypeSignature().toString(),
new ClientTypeSignature(type.getTypeSignature()),
type.getTypeMetadata());
}

public Column(String name, TypeSignature signature)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is what you can do: remove this constructor (Column(String name, TypeSignature signature)). There's no real use case for this one. Create a Type class in client (I think call it Type is fine, but if it feels too confusing we can call it ClientType), which wraps the string type, client type signature and enum maps.

{
this(name, signature.toString(), new ClientTypeSignature(signature));
this(name, signature.toString(), new ClientTypeSignature(signature), null);
}

@JsonCreator
public Column(
@JsonProperty("name") String name,
@JsonProperty("type") String type,
@JsonProperty("typeSignature") ClientTypeSignature typeSignature)
@JsonProperty("typeSignature") ClientTypeSignature typeSignature,
@JsonProperty("typeMetadata") TypeMetadata typeMetadata)
{
this.name = requireNonNull(name, "name is null");
this.type = requireNonNull(type, "type is null");
this.typeSignature = typeSignature;
this.typeMetadata = requireNonNull(typeMetadata, "typeMetadata is null");
}

@JsonProperty
Expand All @@ -67,4 +75,10 @@ public ClientTypeSignature getTypeSignature()
{
return typeSignature;
}

@JsonProperty
public TypeMetadata getTypeMetadata()
{
return typeMetadata;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,10 @@ public static Iterable<List<Object>> fixData(List<Column> columns, Iterable<List
}
requireNonNull(columns, "columns is null");
List<TypeSignature> signatures = columns.stream()
.map(column -> parseTypeSignature(column.getType()))
.map(column -> {
TypeSignature displayType = column.getTypeMetadata().getDisplayType();
Copy link
Contributor Author

@daniel-ohayon daniel-ohayon Jun 22, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in this file, we use the type signature client-side to determine what kind of data we're dealing with.
Since the client doesn't know about the enum types, if it encounters a String value, it will default to base64 encoding as per:

if (value instanceof String) {
    return Base64.getDecoder().decode((String) value);
}

For string-valued enums, this results in gibberish in the CLI output.
This is why I'm passing this extra info about the display type

return displayType != null ? displayType : parseTypeSignature(column.getType());
})
.collect(toList());
ImmutableList.Builder<List<Object>> rows = ImmutableList.builder();
for (List<Object> row : data) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,6 @@
public interface QueryData
{
Iterable<List<Object>> getData();

List<Column> getColumns();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.common.type;

import java.util.Map;

public interface EnumType<T>
extends Type
{
Map<String, T> getEntries();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

getEntries sounds like it should return a list. Maybe explicitly call it getEnumNameToValueMap or just getEnumMap or getMap.


Type getValueType();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.common.type;

import com.facebook.presto.common.block.Block;
import com.facebook.presto.common.function.SqlFunctionProperties;

import java.util.Map;
import java.util.stream.Collectors;

public class IntegerEnumType
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we call it LongEnumType so there's no confusion? Integer can also mean specifically the SQL INTEGER type which is 32-bit (though using 32-bit is probably enough for an enum type so that would be an option as well?)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's go for LongEnumType!

extends AbstractLongType
implements EnumType<Long>
{
private final Map<String, Long> entries;

public IntegerEnumType(String name, Map<String, Long> entries)
{
super(TypeSignature.parseTypeSignature(name));
this.entries = entries;
}

@Override
public Map<String, Long> getEntries()
{
return entries;
}

@Override
public Object getObjectValue(SqlFunctionProperties properties, Block block, int position)
{
if (block.isNull(position)) {
return null;
}

return block.getLong(position);
}

@Override
public boolean isComparable()
{
return false;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Enum types are not comparable?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They should be, but I haven't found a way to register the comparison functions dynamically (like we do for CAST) in a way that I'm happy with yet.

}

@Override
public boolean isOrderable()
{
return false;
}

@Override
public Type getValueType()
{
return BigintType.BIGINT;
}

@Override
public TypeMetadata getTypeMetadata()
{
Map<String, String> stringifiedEntries = this.getEntries().entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
return new TypeMetadata(getValueType().getTypeSignature(), stringifiedEntries);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.common.type;

import java.util.Map;

public class StringEnumType
extends VarcharType
implements EnumType<String>
{
private final Map<String, String> entries;

public StringEnumType(String name, Map<String, String> entries)
{
super(VarcharType.MAX_LENGTH, TypeSignature.parseTypeSignature(name));
this.entries = entries;
}

@Override
public Map<String, String> getEntries()
{
return entries;
}

@Override
public boolean isOrderable()
{
return false;
}

@Override
public boolean isComparable()
{
return false;
}

@Override
public Type getValueType()
{
return VarcharType.VARCHAR;
}

@Override
public TypeMetadata getTypeMetadata()
{
return new TypeMetadata(getValueType().getTypeSignature(), this.getEntries());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -186,4 +186,12 @@ public interface Type
* Compare the values in the specified block at the specified positions equal.
*/
int compareTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition);

/**
* Return additional type information relevant to clients
*/
default TypeMetadata getTypeMetadata()
{
return new TypeMetadata();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.common.type;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;

import javax.annotation.concurrent.Immutable;

import java.util.Map;

@Immutable
public class TypeMetadata
{
private final Map<String, String> enumValues;
private final TypeSignature displayType;

TypeMetadata()
{
this(null, null);
}

@JsonCreator
public TypeMetadata(
@JsonProperty TypeSignature displayType,
@JsonProperty Map<String, String> enumValues)
{
this.displayType = displayType;
this.enumValues = enumValues;
}

@JsonProperty
public TypeSignature getDisplayType()
{
return displayType;
}

@JsonProperty
public Map<String, String> getEnumValues()
{
return enumValues;
}
}
Loading