Skip to content

Commit 41d5627

Browse files
julienledemjulienledem
committed
ARROW-335: Improve Type apis and toString() by encapsulating flatbuffers better
1 parent 121e826 commit 41d5627

File tree

18 files changed

+291
-209
lines changed

18 files changed

+291
-209
lines changed

‎java/vector/src/main/codegen/data/ArrowTypes.tdd‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,15 @@
2626
},
2727
{
2828
name: "Union",
29-
fields: [{name: "mode", type: short}, {name: "typeIds", type: "int[]"}]
29+
fields: [{name: "mode", type: short, valueType: UnionMode}, {name: "typeIds", type: "int[]"}]
3030
},
3131
{
3232
name: "Int",
3333
fields: [{name: "bitWidth", type: int}, {name: "isSigned", type: boolean}]
3434
},
3535
{
3636
name: "FloatingPoint",
37-
fields: [{name: precision, type: short}]
37+
fields: [{name: precision, type: short, valueType: FloatingPointPrecision}]
3838
},
3939
{
4040
name: "Utf8",
@@ -62,11 +62,11 @@
6262
},
6363
{
6464
name: "Timestamp",
65-
fields: [{name: "unit", type: short}]
65+
fields: [{name: "unit", type: short, valueType: TimeUnit}]
6666
},
6767
{
6868
name: "Interval",
69-
fields: [{name: "unit", type: short}]
69+
fields: [{name: "unit", type: short, valueType: IntervalUnit}]
7070
}
7171
]
7272
}

‎java/vector/src/main/codegen/templates/ArrowType.java‎

Lines changed: 38 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -23,30 +23,18 @@
2323
package org.apache.arrow.vector.types.pojo;
2424

2525
import com.google.flatbuffers.FlatBufferBuilder;
26-
import org.apache.arrow.flatbuf.Type;
2726

28-
import java.io.IOException;
2927
import java.util.Objects;
3028

31-
import org.apache.arrow.flatbuf.Precision;
32-
import org.apache.arrow.flatbuf.UnionMode;
33-
import org.apache.arrow.flatbuf.TimeUnit;
34-
import org.apache.arrow.flatbuf.IntervalUnit;
29+
import org.apache.arrow.flatbuf.Type;
30+
31+
import org.apache.arrow.vector.types.*;
3532

3633
import com.fasterxml.jackson.annotation.JsonCreator;
3734
import com.fasterxml.jackson.annotation.JsonIgnore;
3835
import com.fasterxml.jackson.annotation.JsonProperty;
3936
import com.fasterxml.jackson.annotation.JsonSubTypes;
4037
import com.fasterxml.jackson.annotation.JsonTypeInfo;
41-
import com.fasterxml.jackson.core.JsonGenerator;
42-
import com.fasterxml.jackson.core.JsonParser;
43-
import com.fasterxml.jackson.core.JsonProcessingException;
44-
import com.fasterxml.jackson.databind.DeserializationContext;
45-
import com.fasterxml.jackson.databind.JsonDeserializer;
46-
import com.fasterxml.jackson.databind.JsonSerializer;
47-
import com.fasterxml.jackson.databind.SerializerProvider;
48-
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
49-
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
5038

5139
/**
5240
* Arrow types
@@ -57,119 +45,31 @@
5745
property = "name")
5846
@JsonSubTypes({
5947
<#list arrowTypes.types as type>
60-
@JsonSubTypes.Type(value = ArrowType.${type.name}.class, name = "${type.name?remove_ending("_")?lower_case}"),
48+
@JsonSubTypes.Type(value = ArrowType.${type.name?remove_ending("_")}.class, name = "${type.name?remove_ending("_")?lower_case}"),
6149
</#list>
6250
})
6351
public abstract class ArrowType {
6452

65-
private static class FloatingPointPrecisionSerializer extends JsonSerializer<Short> {
66-
@Override
67-
public void serialize(Short precision,
68-
JsonGenerator jsonGenerator,
69-
SerializerProvider serializerProvider)
70-
throws IOException, JsonProcessingException {
71-
jsonGenerator.writeObject(Precision.name(precision));
72-
}
73-
}
74-
75-
private static class FloatingPointPrecisionDeserializer extends JsonDeserializer<Short> {
76-
@Override
77-
public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
78-
String name = p.getText();
79-
switch(name) {
80-
case "HALF":
81-
return Precision.HALF;
82-
case "SINGLE":
83-
return Precision.SINGLE;
84-
case "DOUBLE":
85-
return Precision.DOUBLE;
86-
default:
87-
throw new IllegalArgumentException("unknown precision: " + name);
88-
}
89-
}
90-
}
91-
92-
private static class UnionModeSerializer extends JsonSerializer<Short> {
93-
@Override
94-
public void serialize(Short mode,
95-
JsonGenerator jsonGenerator,
96-
SerializerProvider serializerProvider)
97-
throws IOException, JsonProcessingException {
98-
jsonGenerator.writeObject(UnionMode.name(mode));
99-
}
100-
}
101-
102-
private static class UnionModeDeserializer extends JsonDeserializer<Short> {
103-
@Override
104-
public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
105-
String name = p.getText();
106-
switch(name) {
107-
case "Sparse":
108-
return UnionMode.Sparse;
109-
case "Dense":
110-
return UnionMode.Dense;
111-
default:
112-
throw new IllegalArgumentException("unknown union mode: " + name);
113-
}
114-
}
115-
}
116-
117-
private static class TimestampUnitSerializer extends JsonSerializer<Short> {
118-
@Override
119-
public void serialize(Short unit,
120-
JsonGenerator jsonGenerator,
121-
SerializerProvider serializerProvider)
122-
throws IOException, JsonProcessingException {
123-
jsonGenerator.writeObject(TimeUnit.name(unit));
124-
}
125-
}
126-
127-
private static class TimestampUnitDeserializer extends JsonDeserializer<Short> {
128-
@Override
129-
public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
130-
String name = p.getText();
131-
switch(name) {
132-
case "SECOND":
133-
return TimeUnit.SECOND;
134-
case "MILLISECOND":
135-
return TimeUnit.MILLISECOND;
136-
case "MICROSECOND":
137-
return TimeUnit.MICROSECOND;
138-
case "NANOSECOND":
139-
return TimeUnit.NANOSECOND;
140-
default:
141-
throw new IllegalArgumentException("unknown time unit: " + name);
142-
}
143-
}
144-
}
53+
public static enum ArrowTypeID {
54+
<#list arrowTypes.types as type>
55+
<#assign name = type.name>
56+
${name?remove_ending("_")}(Type.${name}),
57+
</#list>
58+
NONE(Type.NONE);
59+
60+
private final byte flatbufType;
14561

146-
private static class IntervalUnitSerializer extends JsonSerializer<Short> {
147-
@Override
148-
public void serialize(Short unit,
149-
JsonGenerator jsonGenerator,
150-
SerializerProvider serializerProvider)
151-
throws IOException, JsonProcessingException {
152-
jsonGenerator.writeObject(IntervalUnit.name(unit));
62+
public byte getFlatbufID() {
63+
return this.flatbufType;
15364
}
154-
}
15565

156-
private static class IntervalUnitDeserializer extends JsonDeserializer<Short> {
157-
@Override
158-
public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
159-
String name = p.getText();
160-
switch(name) {
161-
case "YEAR_MONTH":
162-
return IntervalUnit.YEAR_MONTH;
163-
case "DAY_TIME":
164-
return IntervalUnit.DAY_TIME;
165-
default:
166-
throw new IllegalArgumentException("unknown interval unit: " + name);
167-
}
66+
private ArrowTypeID(byte flatbufType) {
67+
this.flatbufType = flatbufType;
16868
}
16969
}
17070

17171
@JsonIgnore
172-
public abstract byte getTypeType();
72+
public abstract ArrowTypeID getTypeID();
17373
public abstract int getType(FlatBufferBuilder builder);
17474
public abstract <T> T accept(ArrowTypeVisitor<T> visitor);
17575

@@ -183,28 +83,30 @@ public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOExc
18383
*/
18484
public static interface ArrowTypeVisitor<T> {
18585
<#list arrowTypes.types as type>
186-
T visit(${type.name} type);
86+
T visit(${type.name?remove_ending("_")} type);
18787
</#list>
18888
}
18989

19090
<#list arrowTypes.types as type>
191-
<#assign name = type.name>
91+
<#assign name = type.name?remove_ending("_")>
19292
<#assign fields = type.fields>
19393
public static class ${name} extends ArrowType {
194-
public static final byte TYPE_TYPE = Type.${name};
94+
public static final ArrowTypeID TYPE_TYPE = ArrowTypeID.${name};
19595
<#if type.fields?size == 0>
19696
public static final ${name} INSTANCE = new ${name}();
19797
</#if>
19898

19999
<#list fields as field>
200-
${field.type} ${field.name};
100+
<#assign fieldType = field.valueType!field.type>
101+
${fieldType} ${field.name};
201102
</#list>
202103

203104
<#if type.fields?size != 0>
204105
@JsonCreator
205106
public ${type.name}(
206107
<#list type.fields as field>
207-
<#if field.type == "short"> @JsonDeserialize(using = ${type.name}${field.name?cap_first}Deserializer.class) </#if>@JsonProperty("${field.name}") ${field.type} ${field.name}<#if field_has_next>, </#if>
108+
<#assign fieldType = field.valueType!field.type>
109+
@JsonProperty("${field.name}") ${fieldType} ${field.name}<#if field_has_next>, </#if>
208110
</#list>
209111
) {
210112
<#list type.fields as field>
@@ -214,7 +116,7 @@ public static class ${name} extends ArrowType {
214116
</#if>
215117

216118
@Override
217-
public byte getTypeType() {
119+
public ArrowTypeID getTypeID() {
218120
return TYPE_TYPE;
219121
}
220122

@@ -235,27 +137,29 @@ public int getType(FlatBufferBuilder builder) {
235137
org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, ${field.name});
236138
}
237139
<#else>
238-
org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, this.${field.name});
140+
org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, this.${field.name}<#if field.valueType??>.getFlatbufID()</#if>);
239141
</#if>
240142
</#list>
241143
return org.apache.arrow.flatbuf.${type.name}.end${type.name}(builder);
242144
}
243145

244146
<#list fields as field>
245-
<#if field.type == "short">
246-
@JsonSerialize(using = ${type.name}${field.name?cap_first}Serializer.class)
247-
</#if>
248-
public ${field.type} get${field.name?cap_first}() {
147+
<#assign fieldType = field.valueType!field.type>
148+
public ${fieldType} get${field.name?cap_first}() {
249149
return ${field.name};
250150
}
251151
</#list>
252152

253153
public String toString() {
254-
return "${name}{"
154+
return "${name}"
155+
<#if fields?size != 0>
156+
+ "("
255157
<#list fields as field>
256-
+ <#if field.type == "int[]">java.util.Arrays.toString(${field.name})<#else>${field.name}</#if><#if field_has_next> + ", " </#if>
158+
+ <#if field.type == "int[]">java.util.Arrays.toString(${field.name})<#else>${field.name}</#if><#if field_has_next> + ", " </#if>
257159
</#list>
258-
+ "}";
160+
+ ")"
161+
</#if>
162+
;
259163
}
260164

261165
@Override
@@ -265,7 +169,7 @@ public int hashCode() {
265169

266170
@Override
267171
public boolean equals(Object obj) {
268-
if (!(obj instanceof ${type.name})) {
172+
if (!(obj instanceof ${name})) {
269173
return false;
270174
}
271175
<#if type.fields?size == 0>
@@ -287,7 +191,7 @@ public <T> T accept(ArrowTypeVisitor<T> visitor) {
287191
public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.apache.arrow.flatbuf.Field field) {
288192
switch(field.typeType()) {
289193
<#list arrowTypes.types as type>
290-
<#assign name = type.name>
194+
<#assign name = type.name?remove_ending("_")>
291195
<#assign nameLower = type.name?lower_case>
292196
<#assign fields = type.fields>
293197
case Type.${type.name}: {
@@ -302,7 +206,7 @@ public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.a
302206
${field.type} ${field.name} = ${nameLower}Type.${field.name}();
303207
</#if>
304208
</#list>
305-
return new ${type.name}(<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>);
209+
return new ${name}(<#list type.fields as field><#if field.valueType??>${field.valueType}.fromFlatbufID(${field.name})<#else>${field.name}</#if><#if field_has_next>, </#if></#list>);
306210
}
307211
</#list>
308212
default:

‎java/vector/src/main/codegen/templates/NullableValueVectors.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,15 +99,15 @@ public final class ${className} extends BaseDataValueVector implements <#if type
9999
<#elseif minor.class == "Time">
100100
field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Time(), null);
101101
<#elseif minor.class == "Float4">
102-
field = new Field(name, true, new FloatingPoint(Precision.SINGLE), null);
102+
field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE), null);
103103
<#elseif minor.class == "Float8">
104-
field = new Field(name, true, new FloatingPoint(Precision.DOUBLE), null);
104+
field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE), null);
105105
<#elseif minor.class == "TimeStamp">
106-
field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND), null);
106+
field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND), null);
107107
<#elseif minor.class == "IntervalDay">
108-
field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.DAY_TIME), null);
108+
field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.DAY_TIME), null);
109109
<#elseif minor.class == "IntervalYear">
110-
field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH), null);
110+
field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.YEAR_MONTH), null);
111111
<#elseif minor.class == "VarChar">
112112
field = new Field(name, true, new Utf8(), null);
113113
<#elseif minor.class == "VarBinary">

‎java/vector/src/main/codegen/templates/UnionVector.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@
1515
* See the License for the specific language governing permissions and
1616
* limitations under the License.
1717
*/
18-
import java.util.List;
19-
2018
<@pp.dropOutputFile />
2119
<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/UnionVector.java" />
2220

@@ -35,7 +33,7 @@
3533
import org.apache.arrow.vector.util.CallBack;
3634
import org.apache.arrow.vector.schema.ArrowFieldNode;
3735

38-
import static org.apache.arrow.flatbuf.UnionMode.Sparse;
36+
import static org.apache.arrow.vector.types.UnionMode.Sparse;
3937

4038

4139

‎java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import java.util.Collections;
2121
import java.util.Iterator;
2222

23-
import org.apache.arrow.flatbuf.Type;
2423
import org.apache.arrow.memory.BufferAllocator;
2524
import org.apache.arrow.vector.AddOrGetResult;
2625
import org.apache.arrow.vector.BaseValueVector;
@@ -159,9 +158,9 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorT
159158
created = true;
160159
}
161160

162-
if (vector.getField().getType().getTypeType() != minorType.getType().getTypeType()) {
161+
if (vector.getField().getType().getTypeID() != minorType.getType().getTypeID()) {
163162
final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
164-
Type.name(minorType.getType().getTypeType()), Type.name(vector.getField().getType().getTypeType()));
163+
minorType.getType().getTypeID(), vector.getField().getType().getTypeID());
165164
throw new SchemaChangeRuntimeException(msg);
166165
}
167166

‎java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
import org.apache.arrow.vector.holders.ComplexHolder;
3535
import org.apache.arrow.vector.types.Types;
3636
import org.apache.arrow.vector.types.Types.MinorType;
37-
import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
37+
import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
3838
import org.apache.arrow.vector.types.pojo.Field;
3939
import org.apache.arrow.vector.util.CallBack;
4040
import org.apache.arrow.vector.util.JsonStringHashMap;
@@ -290,7 +290,7 @@ public Field getField() {
290290
for (ValueVector child : getChildren()) {
291291
children.add(child.getField());
292292
}
293-
return new Field(name, false, Struct_.INSTANCE, children);
293+
return new Field(name, false, Struct.INSTANCE, children);
294294
}
295295

296296
@Override

0 commit comments

Comments
 (0)