Skip to content

Commit

Permalink
Add STR_TO_DATE Function To The SQL Plugin (#1420)
Browse files Browse the repository at this point in the history 
* Add `STR_TO_DATE` Function To The SQL Plugin.

* Added Tests

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Added Empty Implementation

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Added Partially Working Implementation

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Cleaned Up Implementation

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Modified Implementation

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Modified IT Test

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Added Documentation

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Added Unit Tests

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Altered Implementation

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Reworked Implementation To Always Return Datetime

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Addressed PR Comments

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Fixed Some Checkstyle Issues

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Cleaned Implementation

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Supported Function Properties

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Fixed Checkstyle

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Removed Unneeded Function

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Cleaned Implementation and Added IT Test

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Fixed Code Coverage and Implementation

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

---------

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

* Cleaned Implementation

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>

---------

Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com>
  • Loading branch information
@GabeFernandez310
GabeFernandez310 authored Mar 16, 2023
1 parent 87018c6 commit 40336d4
Show file tree
Hide file tree
Showing 10 changed files with 453 additions and 8 deletions.
6 changes: 6 additions & 0 deletions core/src/main/java/org/opensearch/sql/expression/DSL.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,12 @@ public static FunctionExpression module(Expression... expressions) {
return compile(FunctionProperties.None, BuiltinFunctionName.MODULES, expressions);
}


public static FunctionExpression str_to_date(FunctionProperties functionProperties,
Expression... expressions) {
return compile(functionProperties, BuiltinFunctionName.STR_TO_DATE, expressions);
}

public static FunctionExpression sec_to_time(Expression... expressions) {
return compile(FunctionProperties.None, BuiltinFunctionName.SEC_TO_TIME, expressions);
}
Expand Down
147 changes: 139 additions & 8 deletions core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,26 @@
package org.opensearch.sql.expression.datetime;

import com.google.common.collect.ImmutableMap;
import java.text.ParsePosition;
import java.time.Clock;
import java.time.DateTimeException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.ResolverStyle;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.opensearch.sql.data.model.ExprDatetimeValue;
import org.opensearch.sql.data.model.ExprNullValue;
import org.opensearch.sql.data.model.ExprStringValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.expression.function.FunctionProperties;

/**
* This class converts a SQL style DATE_FORMAT format specifier and converts it to a
Expand All @@ -28,6 +37,7 @@ class DateTimeFormatterUtil {
private static final String SUFFIX_SPECIAL_TH = "th";

private static final String NANO_SEC_FORMAT = "'%06d'";

private static final Map<Integer, String> SUFFIX_CONVERTER =
ImmutableMap.<Integer, String>builder()
.put(1, "st").put(2, "nd").put(3, "rd").build();
Expand Down Expand Up @@ -122,6 +132,43 @@ interface DateTimeFormatHandler {
.put("%x", (date) -> null)
.build();

private static final Map<String, String> STR_TO_DATE_FORMATS =
ImmutableMap.<String, String>builder()
.put("%a", "EEE") // %a => EEE - Abbreviated weekday name (Sun..Sat)
.put("%b", "LLL") // %b => LLL - Abbreviated month name (Jan..Dec)
.put("%c", "M") // %c => MM - Month, numeric (0..12)
.put("%d", "d") // %d => dd - Day of the month, numeric (00..31)
.put("%e", "d") // %e => d - Day of the month, numeric (0..31)
.put("%H", "H") // %H => HH - (00..23)
.put("%h", "H") // %h => hh - (01..12)
.put("%I", "h") // %I => hh - (01..12)
.put("%i", "m") // %i => mm - Minutes, numeric (00..59)
.put("%j", "DDD") // %j => DDD - (001..366)
.put("%k", "H") // %k => H - (0..23)
.put("%l", "h") // %l => h - (1..12)
.put("%p", "a") // %p => a - AM or PM
.put("%M", "LLLL") // %M => LLLL - Month name (January..December)
.put("%m", "M") // %m => MM - Month, numeric (00..12)
.put("%r", "hh:mm:ss a") // %r => hh:mm:ss a - hh:mm:ss followed by AM or PM
.put("%S", "s") // %S => ss - Seconds (00..59)
.put("%s", "s") // %s => ss - Seconds (00..59)
.put("%T", "HH:mm:ss") // %T => HH:mm:ss
.put("%W", "EEEE") // %W => EEEE - Weekday name (Sunday..Saturday)
.put("%Y", "u") // %Y => yyyy - Year, numeric, 4 digits
.put("%y", "u") // %y => yy - Year, numeric, 2 digits
.put("%f", "n") // %f => n - Nanoseconds
//The following have been implemented but cannot be aligned with
// MySQL due to the limitations of the DatetimeFormatter
.put("%D", "d") // %w - Day of month with English suffix
.put("%w", "e") // %w - Day of week (0 indexed)
.put("%U", "w") // %U Week where Sunday is the first day - WEEK() mode 0
.put("%u", "w") // %u Week where Monday is the first day - WEEK() mode 1
.put("%V", "w") // %V Week where Sunday is the first day - WEEK() mode 2
.put("%v", "w") // %v Week where Monday is the first day - WEEK() mode 3
.put("%X", "u") // %X Year for week where Sunday is the first day
.put("%x", "u") // %x Year for week where Monday is the first day
.build();

private static final Pattern pattern = Pattern.compile("%.");
private static final Pattern CHARACTERS_WITH_NO_MOD_LITERAL_BEHIND_PATTERN
= Pattern.compile("(?<!%)[a-zA-Z&&[^aydmshiHIMYDSEL]]+");
Expand All @@ -130,6 +177,19 @@ interface DateTimeFormatHandler {
private DateTimeFormatterUtil() {
}

static StringBuffer getCleanFormat(ExprValue formatExpr) {
final StringBuffer cleanFormat = new StringBuffer();
final Matcher m = CHARACTERS_WITH_NO_MOD_LITERAL_BEHIND_PATTERN
.matcher(formatExpr.stringValue());

while (m.find()) {
m.appendReplacement(cleanFormat,String.format("'%s'", m.group()));
}
m.appendTail(cleanFormat);

return cleanFormat;
}

/**
* Helper function to format a DATETIME according to a provided handler and matcher.
* @param formatExpr ExprValue containing the format expression
Expand All @@ -140,14 +200,7 @@ private DateTimeFormatterUtil() {
static ExprValue getFormattedString(ExprValue formatExpr,
Map<String, DateTimeFormatHandler> handler,
LocalDateTime datetime) {
final StringBuffer cleanFormat = new StringBuffer();
final Matcher m = CHARACTERS_WITH_NO_MOD_LITERAL_BEHIND_PATTERN
.matcher(formatExpr.stringValue());

while (m.find()) {
m.appendReplacement(cleanFormat,String.format("'%s'", m.group()));
}
m.appendTail(cleanFormat);
StringBuffer cleanFormat = getCleanFormat(formatExpr);

final Matcher matcher = pattern.matcher(cleanFormat.toString());
final StringBuffer format = new StringBuffer();
Expand Down Expand Up @@ -201,6 +254,84 @@ static ExprValue getFormattedTime(ExprValue timeExpr, ExprValue formatExpr) {
return getFormattedString(formatExpr, TIME_HANDLERS, time);
}

private static boolean canGetDate(TemporalAccessor ta) {
return (ta.isSupported(ChronoField.YEAR)
&& ta.isSupported(ChronoField.MONTH_OF_YEAR)
&& ta.isSupported(ChronoField.DAY_OF_MONTH));
}

private static boolean canGetTime(TemporalAccessor ta) {
return (ta.isSupported(ChronoField.HOUR_OF_DAY)
&& ta.isSupported(ChronoField.MINUTE_OF_HOUR)
&& ta.isSupported(ChronoField.SECOND_OF_MINUTE));
}

static ExprValue parseStringWithDateOrTime(FunctionProperties fp,
ExprValue datetimeStringExpr,
ExprValue formatExpr) {

//Replace patterns with % for Java DateTimeFormatter
StringBuffer cleanFormat = getCleanFormat(formatExpr);
final Matcher matcher = pattern.matcher(cleanFormat.toString());
final StringBuffer format = new StringBuffer();

while (matcher.find()) {
matcher.appendReplacement(format,
STR_TO_DATE_FORMATS.getOrDefault(matcher.group(),
String.format("'%s'", matcher.group().replaceFirst(MOD_LITERAL, ""))));
}
matcher.appendTail(format);

TemporalAccessor taWithMissingFields;
//Return NULL for invalid parse in string to align with MySQL
try {
//Get Temporal Accessor to initially parse string without default values
taWithMissingFields = new DateTimeFormatterBuilder()
.appendPattern(format.toString())
.toFormatter().withResolverStyle(ResolverStyle.STRICT)
.parseUnresolved(datetimeStringExpr.stringValue(), new ParsePosition(0));
if (taWithMissingFields == null) {
throw new DateTimeException("Input string could not be parsed properly.");
}
if (!canGetDate(taWithMissingFields) && !canGetTime(taWithMissingFields)) {
throw new DateTimeException("Not enough data to build a valid Date, Time, or Datetime.");
}
} catch (DateTimeException e) {
return ExprNullValue.of();
}

int year = taWithMissingFields.isSupported(ChronoField.YEAR)
? taWithMissingFields.get(ChronoField.YEAR) : 2000;

int month = taWithMissingFields.isSupported(ChronoField.MONTH_OF_YEAR)
? taWithMissingFields.get(ChronoField.MONTH_OF_YEAR) : 1;

int day = taWithMissingFields.isSupported(ChronoField.DAY_OF_MONTH)
? taWithMissingFields.get(ChronoField.DAY_OF_MONTH) : 1;

int hour = taWithMissingFields.isSupported(ChronoField.HOUR_OF_DAY)
? taWithMissingFields.get(ChronoField.HOUR_OF_DAY) : 0;

int minute = taWithMissingFields.isSupported(ChronoField.MINUTE_OF_HOUR)
? taWithMissingFields.get(ChronoField.MINUTE_OF_HOUR) : 0;

int second = taWithMissingFields.isSupported(ChronoField.SECOND_OF_MINUTE)
? taWithMissingFields.get(ChronoField.SECOND_OF_MINUTE) : 0;

//Fill returned datetime with current date if only Time information was parsed
LocalDateTime output;
if (!canGetDate(taWithMissingFields)) {
output = LocalDateTime.of(
LocalDate.now(fp.getQueryStartClock()),
LocalTime.of(hour, minute, second)
);
} else {
output = LocalDateTime.of(year, month, day, hour, minute, second);
}

return new ExprDatetimeValue(output);
}

/**
* Returns English suffix of incoming value.
* @param val Incoming value.
Expand Down
19 changes: 19 additions & 0 deletions core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ public void register(BuiltinFunctionRepository repository) {
repository.register(second(BuiltinFunctionName.SECOND_OF_MINUTE));
repository.register(subdate());
repository.register(subtime());
repository.register(str_to_date());
repository.register(sysdate());
repository.register(time());
repository.register(time_format());
Expand Down Expand Up @@ -810,6 +811,18 @@ private DefaultFunctionResolver subtime() {
);
}

/**
* Extracts a date, time, or datetime from the given string.
* It accomplishes this using another string which specifies the input format.
*/
private DefaultFunctionResolver str_to_date() {
return define(BuiltinFunctionName.STR_TO_DATE.getName(),
implWithProperties(
nullMissingHandlingWithProperties((functionProperties, arg, format)
-> DateTimeFunction.exprStrToDate(functionProperties, arg, format)),
DATETIME, STRING, STRING));
}

/**
* Extracts the time part of a date and time value.
* Also to construct a time type. The supported signatures:
Expand Down Expand Up @@ -1718,6 +1731,12 @@ private ExprValue exprSubTime(FunctionProperties functionProperties,
return exprApplyTime(functionProperties, temporal, temporalDelta, false);
}

private ExprValue exprStrToDate(FunctionProperties fp,
ExprValue dateTimeExpr,
ExprValue formatStringExp) {
return DateTimeFormatterUtil.parseStringWithDateOrTime(fp, dateTimeExpr, formatStringExp);
}

/**
* Time implementation for ExprValue.
*
Expand Down
1 change: 1 addition & 0 deletions core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ public enum BuiltinFunctionName {
SEC_TO_TIME(FunctionName.of("sec_to_time")),
SECOND(FunctionName.of("second")),
SECOND_OF_MINUTE(FunctionName.of("second_of_minute")),
STR_TO_DATE(FunctionName.of("str_to_date")),
SUBDATE(FunctionName.of("subdate")),
SUBTIME(FunctionName.of("subtime")),
TIME(FunctionName.of("time")),
Expand Down
Loading

0 comments on commit 40336d4

Please sign in to comment.