Skip to content

Commit 5217f63

Browse files
martintmartint
committed
Remove usage of TypeSignature in Hive connector
1 parent 7ef8de4 commit 5217f63

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+243
-201
lines changed

‎presto-hive/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,11 @@
185185
<artifactId>javax.inject</artifactId>
186186
</dependency>
187187

188+
<dependency>
189+
<groupId>com.fasterxml.jackson.core</groupId>
190+
<artifactId>jackson-databind</artifactId>
191+
</dependency>
192+
188193
<!-- used by tests but also needed transitively -->
189194
<dependency>
190195
<groupId>io.airlift</groupId>

‎presto-hive/src/main/java/io/prestosql/plugin/hive/GenericHiveRecordCursor.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import io.prestosql.spi.type.DecimalType;
2323
import io.prestosql.spi.type.Decimals;
2424
import io.prestosql.spi.type.Type;
25-
import io.prestosql.spi.type.TypeManager;
2625
import org.apache.hadoop.conf.Configuration;
2726
import org.apache.hadoop.fs.Path;
2827
import org.apache.hadoop.hive.common.type.HiveDecimal;
@@ -121,8 +120,7 @@ public GenericHiveRecordCursor(
121120
long totalBytes,
122121
Properties splitSchema,
123122
List<HiveColumnHandle> columns,
124-
DateTimeZone hiveStorageTimeZone,
125-
TypeManager typeManager)
123+
DateTimeZone hiveStorageTimeZone)
126124
{
127125
requireNonNull(path, "path is null");
128126
requireNonNull(recordReader, "recordReader is null");
@@ -162,7 +160,7 @@ public GenericHiveRecordCursor(
162160
HiveColumnHandle column = columns.get(i);
163161
checkState(column.getColumnType() == REGULAR, "column type must be regular");
164162

165-
types[i] = typeManager.getType(column.getTypeSignature());
163+
types[i] = column.getType();
166164
hiveTypes[i] = column.getHiveType();
167165

168166
StructField field = rowInspector.getStructFieldRef(column.getName());

‎presto-hive/src/main/java/io/prestosql/plugin/hive/GenericHiveRecordCursorProvider.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,7 @@ public Optional<RecordCursor> createRecordCursor(
9393
length,
9494
schema,
9595
columns,
96-
hiveStorageTimeZone,
97-
typeManager));
96+
hiveStorageTimeZone));
9897
});
9998
}
10099

‎presto-hive/src/main/java/io/prestosql/plugin/hive/HiveColumnHandle.java‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,7 @@
1717
import com.fasterxml.jackson.annotation.JsonProperty;
1818
import io.prestosql.spi.connector.ColumnHandle;
1919
import io.prestosql.spi.connector.ColumnMetadata;
20-
import io.prestosql.spi.type.TypeManager;
21-
import io.prestosql.spi.type.TypeSignature;
20+
import io.prestosql.spi.type.Type;
2221

2322
import java.util.Objects;
2423
import java.util.Optional;
@@ -30,6 +29,8 @@
3029
import static io.prestosql.plugin.hive.HiveType.HIVE_LONG;
3130
import static io.prestosql.plugin.hive.HiveType.HIVE_STRING;
3231
import static io.prestosql.spi.type.BigintType.BIGINT;
32+
import static io.prestosql.spi.type.IntegerType.INTEGER;
33+
import static io.prestosql.spi.type.VarcharType.VARCHAR;
3334
import static java.util.Objects.requireNonNull;
3435

3536
public class HiveColumnHandle
@@ -38,22 +39,22 @@ public class HiveColumnHandle
3839
public static final int PATH_COLUMN_INDEX = -11;
3940
public static final String PATH_COLUMN_NAME = "$path";
4041
public static final HiveType PATH_HIVE_TYPE = HIVE_STRING;
41-
public static final TypeSignature PATH_TYPE_SIGNATURE = PATH_HIVE_TYPE.getTypeSignature();
42+
public static final Type PATH_TYPE = VARCHAR;
4243

4344
public static final int BUCKET_COLUMN_INDEX = -12;
4445
public static final String BUCKET_COLUMN_NAME = "$bucket";
4546
public static final HiveType BUCKET_HIVE_TYPE = HIVE_INT;
46-
public static final TypeSignature BUCKET_TYPE_SIGNATURE = BUCKET_HIVE_TYPE.getTypeSignature();
47+
public static final Type BUCKET_TYPE_SIGNATURE = INTEGER;
4748

4849
public static final int FILE_SIZE_COLUMN_INDEX = -13;
4950
public static final String FILE_SIZE_COLUMN_NAME = "$file_size";
5051
public static final HiveType FILE_SIZE_TYPE = HIVE_LONG;
51-
public static final TypeSignature FILE_SIZE_TYPE_SIGNATURE = FILE_SIZE_TYPE.getTypeSignature();
52+
public static final Type FILE_SIZE_TYPE_SIGNATURE = BIGINT;
5253

5354
public static final int FILE_MODIFIED_TIME_COLUMN_INDEX = -14;
5455
public static final String FILE_MODIFIED_TIME_COLUMN_NAME = "$file_modified_time";
5556
public static final HiveType FILE_MODIFIED_TIME_TYPE = HIVE_LONG;
56-
public static final TypeSignature FILE_MODIFIED_TIME_TYPE_SIGNATURE = FILE_SIZE_TYPE.getTypeSignature();
57+
public static final Type FILE_MODIFIED_TIME_TYPE_SIGNATURE = BIGINT;
5758

5859
private static final String UPDATE_ROW_ID_COLUMN_NAME = "$shard_row_id";
5960

@@ -66,7 +67,7 @@ public enum ColumnType
6667

6768
private final String name;
6869
private final HiveType hiveType;
69-
private final TypeSignature typeName;
70+
private final Type type;
7071
private final int hiveColumnIndex;
7172
private final ColumnType columnType;
7273
private final Optional<String> comment;
@@ -75,7 +76,7 @@ public enum ColumnType
7576
public HiveColumnHandle(
7677
@JsonProperty("name") String name,
7778
@JsonProperty("hiveType") HiveType hiveType,
78-
@JsonProperty("typeSignature") TypeSignature typeSignature,
79+
@JsonProperty("type") Type type,
7980
@JsonProperty("hiveColumnIndex") int hiveColumnIndex,
8081
@JsonProperty("columnType") ColumnType columnType,
8182
@JsonProperty("comment") Optional<String> comment)
@@ -84,7 +85,7 @@ public HiveColumnHandle(
8485
checkArgument(hiveColumnIndex >= 0 || columnType == PARTITION_KEY || columnType == SYNTHESIZED, "hiveColumnIndex is negative");
8586
this.hiveColumnIndex = hiveColumnIndex;
8687
this.hiveType = requireNonNull(hiveType, "hiveType is null");
87-
this.typeName = requireNonNull(typeSignature, "type is null");
88+
this.type = requireNonNull(type, "type is null");
8889
this.columnType = requireNonNull(columnType, "columnType is null");
8990
this.comment = requireNonNull(comment, "comment is null");
9091
}
@@ -117,9 +118,9 @@ public boolean isHidden()
117118
return columnType == SYNTHESIZED;
118119
}
119120

120-
public ColumnMetadata getColumnMetadata(TypeManager typeManager)
121+
public ColumnMetadata getColumnMetadata()
121122
{
122-
return new ColumnMetadata(name, typeManager.getType(typeName), null, isHidden());
123+
return new ColumnMetadata(name, type, null, isHidden());
123124
}
124125

125126
@JsonProperty
@@ -129,9 +130,9 @@ public Optional<String> getComment()
129130
}
130131

131132
@JsonProperty
132-
public TypeSignature getTypeSignature()
133+
public Type getType()
133134
{
134-
return typeName;
135+
return type;
135136
}
136137

137138
@JsonProperty
@@ -177,12 +178,12 @@ public static HiveColumnHandle updateRowIdHandle()
177178
// plan-time support for row-by-row delete so that planning doesn't fail. This is why we need
178179
// rowid handle. Note that in Hive connector, rowid handle is not implemented beyond plan-time.
179180

180-
return new HiveColumnHandle(UPDATE_ROW_ID_COLUMN_NAME, HIVE_LONG, BIGINT.getTypeSignature(), -1, SYNTHESIZED, Optional.empty());
181+
return new HiveColumnHandle(UPDATE_ROW_ID_COLUMN_NAME, HIVE_LONG, BIGINT, -1, SYNTHESIZED, Optional.empty());
181182
}
182183

183184
public static HiveColumnHandle pathColumnHandle()
184185
{
185-
return new HiveColumnHandle(PATH_COLUMN_NAME, PATH_HIVE_TYPE, PATH_TYPE_SIGNATURE, PATH_COLUMN_INDEX, SYNTHESIZED, Optional.empty());
186+
return new HiveColumnHandle(PATH_COLUMN_NAME, PATH_HIVE_TYPE, PATH_TYPE, PATH_COLUMN_INDEX, SYNTHESIZED, Optional.empty());
186187
}
187188

188189
/**

‎presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -339,8 +339,8 @@ public HiveTableHandle getTableHandle(ConnectorSession session, SchemaTableName
339339
tableName.getSchemaName(),
340340
tableName.getTableName(),
341341
table.get().getParameters(),
342-
getPartitionKeyColumnHandles(table.get()),
343-
getHiveBucketHandle(table.get()));
342+
getPartitionKeyColumnHandles(table.get(), typeManager),
343+
getHiveBucketHandle(table.get(), typeManager));
344344
}
345345

346346
@Override
@@ -418,14 +418,13 @@ private Optional<SystemTable> getPartitionsSystemTable(ConnectorSession session,
418418
}
419419

420420
List<Type> partitionColumnTypes = partitionColumns.stream()
421-
.map(HiveColumnHandle::getTypeSignature)
422-
.map(typeManager::getType)
421+
.map(HiveColumnHandle::getType)
423422
.collect(toImmutableList());
424423

425424
List<ColumnMetadata> partitionSystemTableColumns = partitionColumns.stream()
426425
.map(column -> new ColumnMetadata(
427426
column.getName(),
428-
typeManager.getType(column.getTypeSignature()),
427+
column.getType(),
429428
column.getComment().orElse(null),
430429
column.isHidden()))
431430
.collect(toImmutableList());
@@ -482,9 +481,9 @@ private ConnectorTableMetadata doGetTableMetadata(ConnectorSession session, Sche
482481
throw new TableNotFoundException(tableName);
483482
}
484483

485-
Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager);
484+
Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get());
486485
ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder();
487-
for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
486+
for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get(), typeManager)) {
488487
columns.add(metadataGetter.apply(columnHandle));
489488
}
490489

@@ -620,7 +619,7 @@ public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, Conn
620619
SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
621620
Table table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName())
622621
.orElseThrow(() -> new TableNotFoundException(tableName));
623-
return hiveColumnHandles(table).stream()
622+
return hiveColumnHandles(table, typeManager).stream()
624623
.collect(toImmutableMap(HiveColumnHandle::getName, identity()));
625624
}
626625

@@ -684,7 +683,7 @@ private List<SchemaTableName> listTables(ConnectorSession session, SchemaTablePr
684683
@Override
685684
public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle)
686685
{
687-
return ((HiveColumnHandle) columnHandle).getColumnMetadata(typeManager);
686+
return ((HiveColumnHandle) columnHandle).getColumnMetadata();
688687
}
689688

690689
@Override
@@ -1087,7 +1086,7 @@ public void finishStatisticsCollection(ConnectorSession session, ConnectorTableH
10871086
List<String> partitionColumnNames = partitionColumns.stream()
10881087
.map(Column::getName)
10891088
.collect(toImmutableList());
1090-
List<HiveColumnHandle> hiveColumnHandles = hiveColumnHandles(table);
1089+
List<HiveColumnHandle> hiveColumnHandles = hiveColumnHandles(table, typeManager);
10911090
Map<String, Type> columnTypes = hiveColumnHandles.stream()
10921091
.filter(columnHandle -> !columnHandle.isHidden())
10931092
.collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager)));
@@ -1115,7 +1114,7 @@ public void finishStatisticsCollection(ConnectorSession session, ConnectorTableH
11151114
Map<String, Set<ColumnStatisticType>> columnStatisticTypes = hiveColumnHandles.stream()
11161115
.filter(columnHandle -> !partitionColumnNames.contains(columnHandle.getName()))
11171116
.filter(column -> !column.isHidden())
1118-
.collect(toImmutableMap(HiveColumnHandle::getName, column -> ImmutableSet.copyOf(metastore.getSupportedColumnStatistics(typeManager.getType(column.getTypeSignature())))));
1117+
.collect(toImmutableMap(HiveColumnHandle::getName, column -> ImmutableSet.copyOf(metastore.getSupportedColumnStatistics(column.getType()))));
11191118
Supplier<PartitionStatistics> emptyPartitionStatistics = Suppliers.memoize(() -> createEmptyPartitionStatistics(columnTypes, columnStatisticTypes));
11201119

11211120
int usedComputedStatistics = 0;
@@ -1389,7 +1388,7 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl
13891388
}
13901389
}
13911390

1392-
List<HiveColumnHandle> handles = hiveColumnHandles(table).stream()
1391+
List<HiveColumnHandle> handles = hiveColumnHandles(table, typeManager).stream()
13931392
.filter(columnHandle -> !columnHandle.isHidden())
13941393
.collect(toList());
13951394

@@ -1958,7 +1957,7 @@ public Optional<ConnectorNewTableLayout> getInsertLayout(ConnectorSession sessio
19581957
}
19591958
}
19601959

1961-
Optional<HiveBucketHandle> hiveBucketHandle = getHiveBucketHandle(table);
1960+
Optional<HiveBucketHandle> hiveBucketHandle = getHiveBucketHandle(table, typeManager);
19621961
if (!hiveBucketHandle.isPresent()) {
19631962
return Optional.empty();
19641963
}
@@ -2206,7 +2205,7 @@ else if (column.isHidden()) {
22062205
columnHandles.add(new HiveColumnHandle(
22072206
column.getName(),
22082207
toHiveType(typeTranslator, column.getType()),
2209-
column.getType().getTypeSignature(),
2208+
column.getType(),
22102209
ordinal,
22112210
columnType,
22122211
Optional.ofNullable(column.getComment())));
@@ -2236,7 +2235,7 @@ private static void validateCsvColumns(ConnectorTableMetadata tableMetadata)
22362235
}
22372236
}
22382237

2239-
private static Function<HiveColumnHandle, ColumnMetadata> columnMetadataGetter(Table table, TypeManager typeManager)
2238+
private static Function<HiveColumnHandle, ColumnMetadata> columnMetadataGetter(Table table)
22402239
{
22412240
ImmutableList.Builder<String> columnNames = ImmutableList.builder();
22422241
table.getPartitionColumns().stream().map(Column::getName).forEach(columnNames::add);
@@ -2269,7 +2268,7 @@ private static Function<HiveColumnHandle, ColumnMetadata> columnMetadataGetter(T
22692268

22702269
return handle -> new ColumnMetadata(
22712270
handle.getName(),
2272-
typeManager.getType(handle.getTypeSignature()),
2271+
handle.getType(),
22732272
columnComment.get(handle.getName()).orElse(null),
22742273
columnExtraInfo(handle.isPartitionKey()),
22752274
handle.isHidden());

‎presto-hive/src/main/java/io/prestosql/plugin/hive/HiveModule.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
*/
1414
package io.prestosql.plugin.hive;
1515

16+
import com.fasterxml.jackson.databind.DeserializationContext;
17+
import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
1618
import com.google.inject.Binder;
1719
import com.google.inject.Module;
1820
import com.google.inject.Provides;
@@ -35,7 +37,11 @@
3537
import io.prestosql.spi.connector.ConnectorPageSinkProvider;
3638
import io.prestosql.spi.connector.ConnectorPageSourceProvider;
3739
import io.prestosql.spi.connector.ConnectorSplitManager;
40+
import io.prestosql.spi.type.Type;
41+
import io.prestosql.spi.type.TypeId;
42+
import io.prestosql.spi.type.TypeManager;
3843

44+
import javax.inject.Inject;
3945
import javax.inject.Singleton;
4046

4147
import java.util.concurrent.ExecutorService;
@@ -45,7 +51,9 @@
4551
import static com.google.inject.multibindings.Multibinder.newSetBinder;
4652
import static io.airlift.concurrent.Threads.daemonThreadsNamed;
4753
import static io.airlift.configuration.ConfigBinder.configBinder;
54+
import static io.airlift.json.JsonBinder.jsonBinder;
4855
import static io.airlift.json.JsonCodecBinder.jsonCodecBinder;
56+
import static java.util.Objects.requireNonNull;
4957
import static java.util.concurrent.Executors.newCachedThreadPool;
5058
import static org.weakref.jmx.guice.ExportBinder.newExporter;
5159

@@ -117,6 +125,8 @@ public void configure(Binder binder)
117125

118126
configBinder(binder).bindConfig(ParquetReaderConfig.class);
119127
configBinder(binder).bindConfig(ParquetWriterConfig.class);
128+
129+
jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
120130
}
121131

122132
@ForHive
@@ -133,4 +143,23 @@ public Function<HiveTransactionHandle, SemiTransactionalHiveMetastore> createMet
133143
{
134144
return transactionHandle -> ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore();
135145
}
146+
147+
public static final class TypeDeserializer
148+
extends FromStringDeserializer<Type>
149+
{
150+
private final TypeManager typeManager;
151+
152+
@Inject
153+
public TypeDeserializer(TypeManager typeManager)
154+
{
155+
super(Type.class);
156+
this.typeManager = requireNonNull(typeManager, "metadata is null");
157+
}
158+
159+
@Override
160+
protected Type _deserialize(String value, DeserializationContext context)
161+
{
162+
return typeManager.getType(TypeId.of(value));
163+
}
164+
}
136165
}

‎presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSink.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import io.prestosql.spi.connector.ConnectorPageSink;
3333
import io.prestosql.spi.connector.ConnectorSession;
3434
import io.prestosql.spi.type.Type;
35-
import io.prestosql.spi.type.TypeManager;
3635
import it.unimi.dsi.fastutil.objects.Object2IntMap;
3736
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
3837

@@ -93,7 +92,6 @@ public HivePageSink(
9392
List<HiveColumnHandle> inputColumns,
9493
Optional<HiveBucketProperty> bucketProperty,
9594
PageIndexerFactory pageIndexerFactory,
96-
TypeManager typeManager,
9795
HdfsEnvironment hdfsEnvironment,
9896
int maxOpenWriters,
9997
ListeningExecutorService writeVerificationExecutor,
@@ -115,8 +113,7 @@ public HivePageSink(
115113
this.pagePartitioner = new HiveWriterPagePartitioner(
116114
inputColumns,
117115
bucketProperty.isPresent(),
118-
pageIndexerFactory,
119-
typeManager);
116+
pageIndexerFactory);
120117

121118
// determine the input index of the partition columns and data columns
122119
// and determine the input index and type of bucketing columns
@@ -401,15 +398,14 @@ private static class HiveWriterPagePartitioner
401398
public HiveWriterPagePartitioner(
402399
List<HiveColumnHandle> inputColumns,
403400
boolean bucketed,
404-
PageIndexerFactory pageIndexerFactory,
405-
TypeManager typeManager)
401+
PageIndexerFactory pageIndexerFactory)
406402
{
407403
requireNonNull(inputColumns, "inputColumns is null");
408404
requireNonNull(pageIndexerFactory, "pageIndexerFactory is null");
409405

410406
List<Type> partitionColumnTypes = inputColumns.stream()
411407
.filter(HiveColumnHandle::isPartitionKey)
412-
.map(column -> typeManager.getType(column.getTypeSignature()))
408+
.map(HiveColumnHandle::getType)
413409
.collect(toList());
414410

415411
if (bucketed) {

‎presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSinkProvider.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,6 @@ private ConnectorPageSink createPageSink(HiveWritableTableHandle handle, boolean
167167
handle.getInputColumns(),
168168
handle.getBucketProperty(),
169169
pageIndexerFactory,
170-
typeManager,
171170
hdfsEnvironment,
172171
maxOpenPartitions,
173172
writeVerificationExecutor,

0 commit comments

Comments
 (0)