Skip to content

Commit f2c6c3e

Browse files
felixYyufelixYyu
authored
Merge branch 'apache:master' into master
2 parents f2bb5d7 + a462ed2 commit f2c6c3e

File tree

18 files changed

+1279
-37
lines changed

18 files changed

+1279
-37
lines changed

‎api/src/main/java/org/apache/iceberg/SortOrder.java

+12-7
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,16 @@ private List<SortField> lazyFieldList() {
135135
return fieldList;
136136
}
137137

138+
UnboundSortOrder toUnbound() {
139+
UnboundSortOrder.Builder builder = UnboundSortOrder.builder().withOrderId(orderId);
140+
141+
for (SortField field : fields) {
142+
builder.addSortField(field.transform().toString(), field.sourceId(), field.direction(), field.nullOrder());
143+
}
144+
145+
return builder.build();
146+
}
147+
138148
@Override
139149
public String toString() {
140150
StringBuilder sb = new StringBuilder();
@@ -244,7 +254,7 @@ public Builder caseSensitive(boolean sortCaseSensitive) {
244254
return this;
245255
}
246256

247-
Builder addSortField(Term term, SortDirection direction, NullOrder nullOrder) {
257+
private Builder addSortField(Term term, SortDirection direction, NullOrder nullOrder) {
248258
Preconditions.checkArgument(term instanceof UnboundTerm, "Term must be unbound");
249259
// ValidationException is thrown by bind if binding fails so we assume that boundTerm is correct
250260
BoundTerm<?> boundTerm = ((UnboundTerm<?>) term).bind(schema.asStruct(), caseSensitive);
@@ -256,18 +266,13 @@ Builder addSortField(Term term, SortDirection direction, NullOrder nullOrder) {
256266

257267
Builder addSortField(String transformAsString, int sourceId, SortDirection direction, NullOrder nullOrder) {
258268
Types.NestedField column = schema.findField(sourceId);
259-
Preconditions.checkNotNull(column, "Cannot find source column: %s", sourceId);
269+
ValidationException.check(column != null, "Cannot find source column: %s", sourceId);
260270
Transform<?, ?> transform = Transforms.fromString(column.type(), transformAsString);
261271
SortField sortField = new SortField(transform, sourceId, direction, nullOrder);
262272
fields.add(sortField);
263273
return this;
264274
}
265275

266-
Builder addSortField(Transform<?, ?> transform, int sourceId, SortDirection direction, NullOrder nullOrder) {
267-
fields.add(new SortField(transform, sourceId, direction, nullOrder));
268-
return this;
269-
}
270-
271276
public SortOrder build() {
272277
SortOrder sortOrder = buildUnchecked();
273278
checkCompatibility(sortOrder, schema);

‎api/src/main/java/org/apache/iceberg/UnboundSortOrder.java

+143
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iceberg;
21+
22+
import java.util.Collections;
23+
import java.util.List;
24+
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
25+
26+
public class UnboundSortOrder {
27+
private static final UnboundSortOrder UNSORTED_ORDER = new UnboundSortOrder(0, Collections.emptyList());
28+
29+
private final int orderId;
30+
private final List<UnboundSortField> fields;
31+
32+
private UnboundSortOrder(int orderId, List<UnboundSortField> fields) {
33+
this.orderId = orderId;
34+
this.fields = fields;
35+
}
36+
37+
public SortOrder bind(Schema schema) {
38+
SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId);
39+
40+
for (UnboundSortField field : fields) {
41+
builder.addSortField(field.transformAsString, field.sourceId, field.direction, field.nullOrder);
42+
}
43+
44+
return builder.build();
45+
}
46+
47+
SortOrder bindUnchecked(Schema schema) {
48+
SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId);
49+
50+
for (UnboundSortField field : fields) {
51+
builder.addSortField(field.transformAsString, field.sourceId, field.direction, field.nullOrder);
52+
}
53+
54+
return builder.buildUnchecked();
55+
}
56+
57+
int orderId() {
58+
return orderId;
59+
}
60+
61+
List<UnboundSortField> fields() {
62+
return fields;
63+
}
64+
65+
/**
66+
* Creates a new {@link SortOrder.Builder sort order builder} for unbound sort orders.
67+
*
68+
* @return a sort order builder
69+
*/
70+
static Builder builder() {
71+
return new Builder();
72+
}
73+
74+
/**
75+
* A builder used to create {@link UnboundSortOrder unbound sort orders}.
76+
* <p>
77+
* Call {@link #builder()} to create a new builder.
78+
*/
79+
static class Builder {
80+
private final List<UnboundSortField> fields = Lists.newArrayList();
81+
private Integer orderId = null;
82+
83+
private Builder() {
84+
}
85+
86+
Builder withOrderId(int newOrderId) {
87+
this.orderId = newOrderId;
88+
return this;
89+
}
90+
91+
Builder addSortField(String transformAsString, int sourceId, SortDirection direction, NullOrder nullOrder) {
92+
fields.add(new UnboundSortField(transformAsString, sourceId, direction, nullOrder));
93+
return this;
94+
}
95+
96+
UnboundSortOrder build() {
97+
if (fields.isEmpty()) {
98+
if (orderId != null && orderId != 0) {
99+
throw new IllegalArgumentException("Unsorted order ID must be 0");
100+
}
101+
return UNSORTED_ORDER;
102+
}
103+
104+
if (orderId != null && orderId == 0) {
105+
throw new IllegalArgumentException("Sort order ID 0 is reserved for unsorted order");
106+
}
107+
108+
// default ID to 1 as 0 is reserved for unsorted order
109+
int actualOrderId = orderId != null ? orderId : 1;
110+
return new UnboundSortOrder(actualOrderId, fields);
111+
}
112+
}
113+
114+
static class UnboundSortField {
115+
private final String transformAsString;
116+
private final int sourceId;
117+
private final SortDirection direction;
118+
private final NullOrder nullOrder;
119+
120+
private UnboundSortField(String transformAsString, int sourceId, SortDirection direction, NullOrder nullOrder) {
121+
this.transformAsString = transformAsString;
122+
this.sourceId = sourceId;
123+
this.direction = direction;
124+
this.nullOrder = nullOrder;
125+
}
126+
127+
public String transformAsString() {
128+
return transformAsString;
129+
}
130+
131+
public int sourceId() {
132+
return sourceId;
133+
}
134+
135+
public SortDirection direction() {
136+
return direction;
137+
}
138+
139+
public NullOrder nullOrder() {
140+
return nullOrder;
141+
}
142+
}
143+
}

‎core/src/main/java/org/apache/iceberg/MetadataUpdate.java

+6-2
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,17 @@ public void applyTo(TableMetadata.Builder metadataBuilder) {
139139
}
140140

141141
class AddSortOrder implements MetadataUpdate {
142-
private final SortOrder sortOrder;
142+
private final UnboundSortOrder sortOrder;
143143

144144
public AddSortOrder(SortOrder sortOrder) {
145+
this(sortOrder.toUnbound());
146+
}
147+
148+
public AddSortOrder(UnboundSortOrder sortOrder) {
145149
this.sortOrder = sortOrder;
146150
}
147151

148-
public SortOrder sortOrder() {
152+
public UnboundSortOrder sortOrder() {
149153
return sortOrder;
150154
}
151155

‎core/src/main/java/org/apache/iceberg/SortOrderParser.java

+54-5
Original file line numberDiff line numberDiff line change
@@ -92,23 +92,72 @@ private static void toJsonFields(SortOrder sortOrder, JsonGenerator generator) t
9292
generator.writeEndArray();
9393
}
9494

95-
public static SortOrder fromJson(Schema schema, String json) {
95+
public static void toJson(UnboundSortOrder sortOrder, JsonGenerator generator) throws IOException {
96+
generator.writeStartObject();
97+
generator.writeNumberField(ORDER_ID, sortOrder.orderId());
98+
generator.writeFieldName(FIELDS);
99+
toJsonFields(sortOrder, generator);
100+
generator.writeEndObject();
101+
}
102+
103+
public static String toJson(UnboundSortOrder sortOrder) {
104+
return toJson(sortOrder, false);
105+
}
106+
107+
public static String toJson(UnboundSortOrder sortOrder, boolean pretty) {
96108
try {
97-
return fromJson(schema, JsonUtil.mapper().readValue(json, JsonNode.class));
109+
StringWriter writer = new StringWriter();
110+
JsonGenerator generator = JsonUtil.factory().createGenerator(writer);
111+
if (pretty) {
112+
generator.useDefaultPrettyPrinter();
113+
}
114+
toJson(sortOrder, generator);
115+
generator.flush();
116+
return writer.toString();
117+
98118
} catch (IOException e) {
99119
throw new UncheckedIOException(e);
100120
}
101121
}
102122

123+
private static void toJsonFields(UnboundSortOrder sortOrder, JsonGenerator generator) throws IOException {
124+
generator.writeStartArray();
125+
for (UnboundSortOrder.UnboundSortField field : sortOrder.fields()) {
126+
generator.writeStartObject();
127+
generator.writeStringField(TRANSFORM, field.transformAsString());
128+
generator.writeNumberField(SOURCE_ID, field.sourceId());
129+
generator.writeStringField(DIRECTION, toJson(field.direction()));
130+
generator.writeStringField(NULL_ORDER, toJson(field.nullOrder()));
131+
generator.writeEndObject();
132+
}
133+
generator.writeEndArray();
134+
}
135+
136+
public static SortOrder fromJson(Schema schema, String json) {
137+
return fromJson(json).bind(schema);
138+
}
139+
103140
public static SortOrder fromJson(Schema schema, JsonNode json) {
141+
return fromJson(json).bind(schema);
142+
}
143+
144+
public static UnboundSortOrder fromJson(String json) {
145+
try {
146+
return fromJson(JsonUtil.mapper().readValue(json, JsonNode.class));
147+
} catch (IOException e) {
148+
throw new UncheckedIOException(e);
149+
}
150+
}
151+
152+
public static UnboundSortOrder fromJson(JsonNode json) {
104153
Preconditions.checkArgument(json.isObject(), "Cannot parse sort order from non-object: %s", json);
105154
int orderId = JsonUtil.getInt(ORDER_ID, json);
106-
SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId);
155+
UnboundSortOrder.Builder builder = UnboundSortOrder.builder().withOrderId(orderId);
107156
buildFromJsonFields(builder, json.get(FIELDS));
108157
return builder.build();
109158
}
110159

111-
private static void buildFromJsonFields(SortOrder.Builder builder, JsonNode json) {
160+
private static void buildFromJsonFields(UnboundSortOrder.Builder builder, JsonNode json) {
112161
Preconditions.checkArgument(json != null, "Cannot parse null sort order fields");
113162
Preconditions.checkArgument(json.isArray(), "Cannot parse sort order fields, not an array: %s", json);
114163

@@ -135,7 +184,7 @@ private static SortDirection toDirection(String directionAsString) {
135184
}
136185

137186
private static NullOrder toNullOrder(String nullOrderingAsString) {
138-
switch (nullOrderingAsString) {
187+
switch (nullOrderingAsString.toLowerCase(Locale.ROOT)) {
139188
case "nulls-first":
140189
return NULLS_FIRST;
141190
case "nulls-last":

‎core/src/main/java/org/apache/iceberg/TableMetadata.java

+8-10
Original file line numberDiff line numberDiff line change
@@ -644,16 +644,9 @@ private static PartitionSpec updateSpecSchema(Schema schema, PartitionSpec parti
644644
}
645645

646646
private static SortOrder updateSortOrderSchema(Schema schema, SortOrder sortOrder) {
647-
SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(sortOrder.orderId());
648-
649-
// add all the fields to the builder. IDs should not change.
650-
for (SortField field : sortOrder.fields()) {
651-
builder.addSortField(field.transform(), field.sourceId(), field.direction(), field.nullOrder());
652-
}
653-
654647
// build without validation because the schema may have changed in a way that makes this order invalid. the order
655648
// should still be preserved so that older metadata can be interpreted.
656-
return builder.buildUnchecked();
649+
return sortOrder.toUnbound().bindUnchecked(schema);
657650
}
658651

659652
private static PartitionSpec freshSpec(int specId, Schema schema, PartitionSpec partitionSpec) {
@@ -674,7 +667,7 @@ private static PartitionSpec freshSpec(int specId, Schema schema, PartitionSpec
674667
}
675668

676669
private static SortOrder freshSortOrder(int orderId, Schema schema, SortOrder sortOrder) {
677-
SortOrder.Builder builder = SortOrder.builderFor(schema);
670+
UnboundSortOrder.Builder builder = UnboundSortOrder.builder();
678671

679672
if (sortOrder.isSorted()) {
680673
builder.withOrderId(orderId);
@@ -692,7 +685,7 @@ private static SortOrder freshSortOrder(int orderId, Schema schema, SortOrder so
692685
field.nullOrder());
693686
}
694687

695-
return builder.build();
688+
return builder.build().bind(schema);
696689
}
697690

698691
private static Map<Long, Snapshot> indexAndValidateSnapshots(List<Snapshot> snapshots, long lastSequenceNumber) {
@@ -964,6 +957,11 @@ public Builder setDefaultSortOrder(int sortOrderId) {
964957
return this;
965958
}
966959

960+
public Builder addSortOrder(UnboundSortOrder order) {
961+
addSortOrderInternal(order.bind(schemasById.get(currentSchemaId)));
962+
return this;
963+
}
964+
967965
public Builder addSortOrder(SortOrder order) {
968966
addSortOrderInternal(order);
969967
return this;

‎core/src/main/java/org/apache/iceberg/rest/RESTSerializers.java

+8-9
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@
3333
import org.apache.iceberg.PartitionSpecParser;
3434
import org.apache.iceberg.Schema;
3535
import org.apache.iceberg.SchemaParser;
36-
import org.apache.iceberg.SortOrder;
3736
import org.apache.iceberg.SortOrderParser;
37+
import org.apache.iceberg.UnboundSortOrder;
3838
import org.apache.iceberg.catalog.Namespace;
3939
import org.apache.iceberg.catalog.TableIdentifier;
4040
import org.apache.iceberg.catalog.TableIdentifierParser;
@@ -56,8 +56,8 @@ public static void registerAll(ObjectMapper mapper) {
5656
.addDeserializer(Schema.class, new SchemaDeserializer())
5757
.addSerializer(PartitionSpec.class, new PartitionSpecSerializer())
5858
.addDeserializer(PartitionSpec.class, new PartitionSpecDeserializer())
59-
.addSerializer(SortOrder.class, new SortOrderSerializer())
60-
.addDeserializer(SortOrder.class, new SortOrderDeserializer());
59+
.addSerializer(UnboundSortOrder.class, new UnboundSortOrderSerializer())
60+
.addDeserializer(UnboundSortOrder.class, new UnboundSortOrderDeserializer());
6161
mapper.registerModule(module);
6262
}
6363

@@ -133,20 +133,19 @@ public PartitionSpec deserialize(JsonParser p, DeserializationContext context)
133133
}
134134
}
135135

136-
public static class SortOrderSerializer extends JsonSerializer<SortOrder> {
136+
public static class UnboundSortOrderSerializer extends JsonSerializer<UnboundSortOrder> {
137137
@Override
138-
public void serialize(SortOrder sortOrder, JsonGenerator gen, SerializerProvider serializers)
138+
public void serialize(UnboundSortOrder sortOrder, JsonGenerator gen, SerializerProvider serializers)
139139
throws IOException {
140140
SortOrderParser.toJson(sortOrder, gen);
141141
}
142142
}
143143

144-
public static class SortOrderDeserializer extends JsonDeserializer<SortOrder> {
144+
public static class UnboundSortOrderDeserializer extends JsonDeserializer<UnboundSortOrder> {
145145
@Override
146-
public SortOrder deserialize(JsonParser p, DeserializationContext context) throws IOException {
146+
public UnboundSortOrder deserialize(JsonParser p, DeserializationContext context) throws IOException {
147147
JsonNode jsonNode = p.getCodec().readTree(p);
148-
Schema schema = (Schema) context.getAttribute("schema");
149-
return SortOrderParser.fromJson(schema, jsonNode);
148+
return SortOrderParser.fromJson(jsonNode);
150149
}
151150
}
152151
}

0 commit comments

Comments
 (0)