Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

API, Core: implement types timestamp_ns and timestamptz_ns #8971

Closed
wants to merge 9 commits into from
Closed

API, Core: implement types timestamp_ns and timestamptz_ns #8971

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3c77542
API: implement types timestamp_ns and timestamptz_ns
jacobmarble Sep 26, 2023
5fd82c2
Core: add types timestamp_ns and timestamptz_ns
jacobmarble Sep 28, 2023
24e6143
chore: update ORC and Parquet
jacobmarble Nov 3, 2023
3b87435
chore: cast timestamp and get unit once
jacobmarble Nov 3, 2023
74b90d9
chore: clean up switch/default/if/else logic
jacobmarble Nov 3, 2023
5278960
chore: cleanup exceptions thrown for timestamp unit
jacobmarble Nov 3, 2023
77bd16d
chore: deprecate TimestampType.with[out]Zone()
jacobmarble Nov 3, 2023
c910484
fix: interpret timestamp strings as micros or nanos
jacobmarble Nov 6, 2023
cb109a4
cleanup: revert unrelated change
jacobmarble Nov 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: interpret timestamp strings as micros or nanos 
Resolves:
- #8971 (comment)
  • Loading branch information
@jacobmarble
jacobmarble committed Nov 7, 2023
commit c9104845f70ad871b1f04a1b3dfaa6459f72a2d7
21 changes: 15 additions & 6 deletions api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,15 @@ public class ExpressionUtil {
private static final Pattern DATE = Pattern.compile("\\d{4}-\\d{2}-\\d{2}");
private static final Pattern TIME = Pattern.compile("\\d{2}:\\d{2}(:\\d{2}(.\\d{1,9})?)?");
private static final Pattern TIMESTAMP =
Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}(:\\d{2}(.\\d{1,9})?)?");
Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}(:\\d{2}(.\\d{1,6})?)?");
private static final Pattern TIMESTAMPNS =
Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}(:\\d{2}(.\\d{7,9})?)?");
private static final Pattern TIMESTAMPTZ =
Pattern.compile(
"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}(:\\d{2}(.\\d{1,9})?)?([-+]\\d{2}:\\d{2}|Z)");
"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}(:\\d{2}(.\\d{1,6})?)?([-+]\\d{2}:\\d{2}|Z)");
private static final Pattern TIMESTAMPTZNS =
Pattern.compile(
"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}(:\\d{2}(.\\d{7,9})?)?([-+]\\d{2}:\\d{2}|Z)");
static final int LONG_IN_PREDICATE_ABBREVIATION_THRESHOLD = 10;
private static final int LONG_IN_PREDICATE_ABBREVIATION_MIN_GAIN = 5;

Expand Down Expand Up @@ -534,10 +539,8 @@ private static String sanitize(Literal<?> literal, long nowMillis, int today) {
} else if (literal instanceof Literals.DateLiteral) {
return sanitizeDate(((Literals.DateLiteral) literal).value(), today);
} else if (literal instanceof Literals.TimestampLiteral) {
return sanitizeTimestamp(
((Literals.TimestampLiteral) literal).unit(),
((Literals.TimestampLiteral) literal).value(),
nowMillis);
Literals.TimestampLiteral tsLiteral = ((Literals.TimestampLiteral) literal);
return sanitizeTimestamp(tsLiteral.unit(), tsLiteral.value(), nowMillis);
} else if (literal instanceof Literals.TimeLiteral) {
return "(time)";
} else if (literal instanceof Literals.IntegerLiteral) {
Expand Down Expand Up @@ -609,9 +612,15 @@ private static String sanitizeString(CharSequence value, long nowMillis, int tod
Literal<Integer> date = Literal.of(value).to(Types.DateType.get());
return sanitizeDate(date.value(), today);
} else if (TIMESTAMP.matcher(value).matches()) {
Literal<Long> ts = Literal.of(value).to(Types.TimestampType.microsWithoutZone());
return sanitizeTimestamp(ChronoUnit.MICROS, ts.value(), nowMillis);
} else if (TIMESTAMPNS.matcher(value).matches()) {
Literal<Long> ts = Literal.of(value).to(Types.TimestampType.nanosWithoutZone());
return sanitizeTimestamp(ChronoUnit.NANOS, ts.value(), nowMillis);
} else if (TIMESTAMPTZ.matcher(value).matches()) {
Literal<Long> ts = Literal.of(value).to(Types.TimestampType.microsWithZone());
return sanitizeTimestamp(ChronoUnit.MICROS, ts.value(), nowMillis);
} else if (TIMESTAMPTZNS.matcher(value).matches()) {
Literal<Long> ts = Literal.of(value).to(Types.TimestampType.nanosWithZone());
return sanitizeTimestamp(ChronoUnit.NANOS, ts.value(), nowMillis);
} else if (TIME.matcher(value).matches()) {
Expand Down
71 changes: 65 additions & 6 deletions api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,12 @@ public class TestExpressionUtil {
Types.NestedField.required(2, "val", Types.IntegerType.get()),
Types.NestedField.required(3, "val2", Types.IntegerType.get()),
Types.NestedField.required(4, "ts", Types.TimestampType.microsWithoutZone()),
Types.NestedField.required(5, "date", Types.DateType.get()),
Types.NestedField.required(6, "time", Types.DateType.get()),
Types.NestedField.optional(7, "data", Types.StringType.get()),
Types.NestedField.optional(8, "measurement", Types.DoubleType.get()),
Types.NestedField.optional(9, "test", Types.IntegerType.get()));
Types.NestedField.required(5, "tsns", Types.TimestampType.nanosWithoutZone()),
Types.NestedField.required(6, "date", Types.DateType.get()),
Types.NestedField.required(7, "time", Types.DateType.get()),
Types.NestedField.optional(8, "data", Types.StringType.get()),
Types.NestedField.optional(9, "measurement", Types.DoubleType.get()),
Types.NestedField.optional(10, "test", Types.IntegerType.get()));

private static final Types.StructType STRUCT = SCHEMA.asStruct();

Expand Down Expand Up @@ -461,7 +462,9 @@ public void testSanitizeTimestamp() {
"2022-04-29T23:49:51",
"2022-04-29T23:49:51.123456",
"2022-04-29T23:49:51-07:00",
"2022-04-29T23:49:51.123456+01:00")) {
"2022-04-29T23:49:51.123456+01:00",
"2022-04-29T23:49:51.123456789",
"2022-04-29T23:49:51.123456789+01:00")) {
assertEquals(
Expressions.equal("test", "(timestamp)"),
ExpressionUtil.sanitize(Expressions.equal("test", timestamp)));
Expand Down Expand Up @@ -497,6 +500,13 @@ public void testSanitizeTimestampAboutNow() {
Expression.Operation.EQ,
"test",
Literal.of(nowLocal).to(Types.TimestampType.microsWithoutZone()))));
assertEquals(
Expressions.equal("test", "(timestamp-about-now)"),
ExpressionUtil.sanitize(
Expressions.predicate(
Expression.Operation.EQ,
"test",
Literal.of(nowLocal).to(Types.TimestampType.nanosWithoutZone()))));

assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", nowLocal)))
.as("Sanitized string should be identical except for descriptive literal")
Expand All @@ -523,6 +533,13 @@ public void testSanitizeTimestampPast() {
Expression.Operation.EQ,
"test",
Literal.of(ninetyMinutesAgoLocal).to(Types.TimestampType.microsWithoutZone()))));
assertEquals(
Expressions.equal("test", "(timestamp-1-hours-ago)"),
ExpressionUtil.sanitize(
Expressions.predicate(
Expression.Operation.EQ,
"test",
Literal.of(ninetyMinutesAgoLocal).to(Types.TimestampType.nanosWithoutZone()))));

assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", ninetyMinutesAgoLocal)))
.as("Sanitized string should be identical except for descriptive literal")
Expand All @@ -549,6 +566,13 @@ public void testSanitizeTimestampLastWeek() {
Expression.Operation.EQ,
"test",
Literal.of(lastWeekLocal).to(Types.TimestampType.microsWithoutZone()))));
assertEquals(
Expressions.equal("test", "(timestamp-7-days-ago)"),
ExpressionUtil.sanitize(
Expressions.predicate(
Expression.Operation.EQ,
"test",
Literal.of(lastWeekLocal).to(Types.TimestampType.nanosWithoutZone()))));

assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", lastWeekLocal)))
.as("Sanitized string should be identical except for descriptive literal")
Expand Down Expand Up @@ -576,6 +600,13 @@ public void testSanitizeTimestampFuture() {
"test",
Literal.of(ninetyMinutesFromNowLocal)
.to(Types.TimestampType.microsWithoutZone()))));
assertEquals(
Expressions.equal("test", "(timestamp-1-hours-from-now)"),
ExpressionUtil.sanitize(
Expressions.predicate(
Expression.Operation.EQ,
"test",
Literal.of(ninetyMinutesFromNowLocal).to(Types.TimestampType.nanosWithoutZone()))));

assertThat(
ExpressionUtil.toSanitizedString(Expressions.equal("test", ninetyMinutesFromNowLocal)))
Expand All @@ -599,6 +630,13 @@ public void testSanitizeTimestamptzAboutNow() {
Expression.Operation.EQ,
"test",
Literal.of(nowUtc).to(Types.TimestampType.microsWithZone()))));
assertEquals(
Expressions.equal("test", "(timestamp-about-now)"),
ExpressionUtil.sanitize(
Expressions.predicate(
Expression.Operation.EQ,
"test",
Literal.of(nowUtc).to(Types.TimestampType.nanosWithZone()))));

assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", nowUtc)))
.as("Sanitized string should be identical except for descriptive literal")
Expand All @@ -620,6 +658,13 @@ public void testSanitizeTimestamptzPast() {
Expression.Operation.EQ,
"test",
Literal.of(ninetyMinutesAgoUtc).to(Types.TimestampType.microsWithZone()))));
assertEquals(
Expressions.equal("test", "(timestamp-1-hours-ago)"),
ExpressionUtil.sanitize(
Expressions.predicate(
Expression.Operation.EQ,
"test",
Literal.of(ninetyMinutesAgoUtc).to(Types.TimestampType.nanosWithZone()))));

assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", ninetyMinutesAgoUtc)))
.as("Sanitized string should be identical except for descriptive literal")
Expand All @@ -641,6 +686,13 @@ public void testSanitizeTimestamptzLastWeek() {
Expression.Operation.EQ,
"test",
Literal.of(lastWeekUtc).to(Types.TimestampType.microsWithZone()))));
assertEquals(
Expressions.equal("test", "(timestamp-7-days-ago)"),
ExpressionUtil.sanitize(
Expressions.predicate(
Expression.Operation.EQ,
"test",
Literal.of(lastWeekUtc).to(Types.TimestampType.nanosWithZone()))));

assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", lastWeekUtc)))
.as("Sanitized string should be identical except for descriptive literal")
Expand All @@ -662,6 +714,13 @@ public void testSanitizeTimestamptzFuture() {
Expression.Operation.EQ,
"test",
Literal.of(ninetyMinutesFromNowUtc).to(Types.TimestampType.microsWithZone()))));
assertEquals(
Expressions.equal("test", "(timestamp-1-hours-from-now)"),
ExpressionUtil.sanitize(
Expressions.predicate(
Expression.Operation.EQ,
"test",
Literal.of(ninetyMinutesFromNowUtc).to(Types.TimestampType.nanosWithZone()))));

assertThat(ExpressionUtil.toSanitizedString(Expressions.equal("test", ninetyMinutesFromNowUtc)))
.as("Sanitized string should be identical except for descriptive literal")
Expand Down