Add The TO_SECONDS Function To The SQL Plugin (#1419)

* Add The `TO_SECONDS` Function To The SQL Plugin (#232) * Added Basic Tests Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Added IT Test Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Added Implementation Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Changed Integration Tests Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Added Test For Time Type Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Added Documentation Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Addressed PR Comments Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Fixed Docs and Implementation Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Fixed Checkstyle Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Changed DateTimeFormatter Priority Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Added More Formatters Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Updated Docs Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Reworked Implementation For Formatters Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Cleanup Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Added Test Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Fixed Implementation And Code Coverage Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Changed getFormatter Function Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Added Comments Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> --------- Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> * Removed Unneeded Code Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> --------- Signed-off-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> (cherry picked from commit d38a6ec)
opensearch-project · Mar 16, 2023 · e1b5713 · e1b5713
1 parent 5f1747d
commit e1b5713
Show file tree

Hide file tree

Showing 10 changed files with 317 additions and 1 deletion.
diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java
@@ -314,6 +314,10 @@ public static FunctionExpression datetime(Expression... expressions) {
     return compile(FunctionProperties.None, BuiltinFunctionName.DATETIME, expressions);
   }
 
+  public static FunctionExpression date_add(Expression... expressions) {
+    return compile(FunctionProperties.None, BuiltinFunctionName.DATE_ADD, expressions);
+  }
+
   public static FunctionExpression day(Expression... expressions) {
     return compile(FunctionProperties.None, BuiltinFunctionName.DAY, expressions);
   }
@@ -451,6 +455,16 @@ public static FunctionExpression to_days(Expression... expressions) {
     return compile(FunctionProperties.None, BuiltinFunctionName.TO_DAYS, expressions);
   }
 
+  public static FunctionExpression to_seconds(FunctionProperties functionProperties,
+                                              Expression... expressions) {
+    return compile(functionProperties, BuiltinFunctionName.TO_SECONDS, expressions);
+  }
+
+  public static FunctionExpression to_seconds(Expression... expressions) {
+    return to_seconds(FunctionProperties.None, expressions);
+  }
+
+
   public static FunctionExpression week(
       FunctionProperties functionProperties, Expression... expressions) {
     return compile(functionProperties, BuiltinFunctionName.WEEK, expressions);

diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java
@@ -28,10 +28,18 @@
 import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling;
 import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties;
 import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_LONG_YEAR;
+import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_NO_YEAR;
 import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_SHORT_YEAR;
+import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_SINGLE_DIGIT_MONTH;
+import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_SINGLE_DIGIT_YEAR;
 import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_LONG_YEAR;
 import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_SHORT_YEAR;
 import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_STRICT_WITH_TZ;
+import static org.opensearch.sql.utils.DateTimeFormatters.FULL_DATE_LENGTH;
+import static org.opensearch.sql.utils.DateTimeFormatters.NO_YEAR_DATE_LENGTH;
+import static org.opensearch.sql.utils.DateTimeFormatters.SHORT_DATE_LENGTH;
+import static org.opensearch.sql.utils.DateTimeFormatters.SINGLE_DIGIT_MONTH_DATE_LENGTH;
+import static org.opensearch.sql.utils.DateTimeFormatters.SINGLE_DIGIT_YEAR_DATE_LENGTH;
 import static org.opensearch.sql.utils.DateTimeUtils.extractDate;
 import static org.opensearch.sql.utils.DateTimeUtils.extractDateTime;
 
@@ -97,6 +105,9 @@
 @UtilityClass
 @SuppressWarnings("unchecked")
 public class DateTimeFunction {
+  //The number of seconds per day
+  public static final long SECONDS_PER_DAY = 86400;
+
   // The number of days from year zero to year 1970.
   private static final Long DAYS_0000_TO_1970 = (146097 * 5L) - (30L * 365L + 7L);
 
@@ -107,7 +118,6 @@ public class DateTimeFunction {
   // Mode used for week/week_of_year function by default when no argument is provided
   private static final ExprIntegerValue DEFAULT_WEEK_OF_YEAR_MODE = new ExprIntegerValue(0);
 
-
   // Map used to determine format output for the extract function
   private static final Map<String, String> extract_formats =
       ImmutableMap.<String, String>builder()
@@ -224,6 +234,7 @@ public void register(BuiltinFunctionRepository repository) {
     repository.register(utc_timestamp());
     repository.register(date_format());
     repository.register(to_days());
+    repository.register(to_seconds());
     repository.register(unix_timestamp());
     repository.register(week(BuiltinFunctionName.WEEK));
     repository.register(week(BuiltinFunctionName.WEEKOFYEAR));
@@ -893,6 +904,17 @@ private DefaultFunctionResolver to_days() {
         impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, DATETIME));
   }
 
+  /**
+   * TO_SECONDS(TIMESTAMP/LONG). return the seconds number of the given date.
+   * Arguments of type STRING/TIMESTAMP/LONG are also accepted.
+   * STRING/TIMESTAMP/LONG arguments are automatically cast to TIMESTAMP.
+   */
+  private DefaultFunctionResolver to_seconds() {
+    return define(BuiltinFunctionName.TO_SECONDS.getName(),
+        impl(nullMissingHandling(DateTimeFunction::exprToSeconds), LONG, TIMESTAMP),
+        impl(nullMissingHandling(DateTimeFunction::exprToSecondsForIntType), LONG, LONG));
+  }
+
   private FunctionResolver unix_timestamp() {
     return define(BuiltinFunctionName.UNIX_TIMESTAMP.getName(),
         implWithProperties(functionProperties
@@ -1816,6 +1838,80 @@ private ExprValue exprToDays(ExprValue date) {
     return new ExprLongValue(date.dateValue().toEpochDay() + DAYS_0000_TO_1970);
   }
 
+  /**
+   * To_seconds implementation for ExprValue.
+   *
+   * @param date ExprValue of Date/Datetime/Timestamp/String type.
+   * @return ExprValue.
+   */
+  private ExprValue exprToSeconds(ExprValue date) {
+    return new ExprLongValue(
+        date.datetimeValue().toEpochSecond(ZoneOffset.UTC) + DAYS_0000_TO_1970 * SECONDS_PER_DAY);
+  }
+
+  /**
+   * Helper function to determine the correct formatter for date arguments passed in as integers.
+   *
+   * @param dateAsInt is an integer formatted as one of YYYYMMDD, YYMMDD, YMMDD, MMDD, MDD
+   * @return is a DateTimeFormatter that can parse the input.
+   */
+  private DateTimeFormatter getFormatter(int dateAsInt) {
+    int length = String.format("%d", dateAsInt).length();
+
+    if (length > 8) {
+      throw new DateTimeException("Integer argument was out of range");
+    }
+
+    //Check below from YYYYMMDD - MMDD which format should be used
+    switch (length) {
+      //Check if dateAsInt is at least 8 digits long
+      case FULL_DATE_LENGTH:
+        return DATE_FORMATTER_LONG_YEAR;
+
+      //Check if dateAsInt is at least 6 digits long
+      case SHORT_DATE_LENGTH:
+        return DATE_FORMATTER_SHORT_YEAR;
+
+      //Check if dateAsInt is at least 5 digits long
+      case SINGLE_DIGIT_YEAR_DATE_LENGTH:
+        return DATE_FORMATTER_SINGLE_DIGIT_YEAR;
+
+      //Check if dateAsInt is at least 4 digits long
+      case NO_YEAR_DATE_LENGTH:
+        return DATE_FORMATTER_NO_YEAR;
+
+      //Check if dateAsInt is at least 3 digits long
+      case SINGLE_DIGIT_MONTH_DATE_LENGTH:
+        return DATE_FORMATTER_SINGLE_DIGIT_MONTH;
+
+      default:
+        break;
+    }
+
+    throw new DateTimeException("No Matching Format");
+  }
+
+  /**
+   * To_seconds implementation with an integer argument for ExprValue.
+   *
+   * @param dateExpr ExprValue of an Integer/Long formatted for a date (e.g., 950501 = 1995-05-01)
+   * @return ExprValue.
+   */
+  private ExprValue exprToSecondsForIntType(ExprValue dateExpr) {
+    try {
+      //Attempt to parse integer argument as date
+      LocalDate date = LocalDate.parse(String.valueOf(dateExpr.integerValue()),
+          getFormatter(dateExpr.integerValue()));
+
+      return new ExprLongValue(date.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC)
+          + DAYS_0000_TO_1970 * SECONDS_PER_DAY);
+
+    } catch (DateTimeException ignored) {
+      //Return null if parsing error
+      return ExprNullValue.of();
+    }
+  }
+
   /**
    * Week for date implementation for ExprValue.
    *

diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java
@@ -109,6 +109,7 @@ public enum BuiltinFunctionName {
   TIMESTAMP(FunctionName.of("timestamp")),
   TIME_FORMAT(FunctionName.of("time_format")),
   TO_DAYS(FunctionName.of("to_days")),
+  TO_SECONDS(FunctionName.of("to_seconds")),
   UTC_DATE(FunctionName.of("utc_date")),
   UTC_TIME(FunctionName.of("utc_time")),
   UTC_TIMESTAMP(FunctionName.of("utc_timestamp")),

diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java
@@ -29,6 +29,21 @@
 @UtilityClass
 public class DateTimeFormatters {
 
+  //Length of a date formatted as YYYYMMDD.
+  public static final int FULL_DATE_LENGTH = 8;
+
+  //Length of a date formatted as YYMMDD.
+  public static final int SHORT_DATE_LENGTH = 6;
+
+  //Length of a date formatted as YMMDD.
+  public static final int SINGLE_DIGIT_YEAR_DATE_LENGTH = 5;
+
+  //Length of a date formatted as MMDD.
+  public static final int NO_YEAR_DATE_LENGTH = 4;
+
+  //Length of a date formatted as MDD.
+  public static final int SINGLE_DIGIT_MONTH_DATE_LENGTH = 3;
+
   public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON =
       new DateTimeFormatterBuilder()
           .appendOffset("+HHmm", "Z")
@@ -133,6 +148,30 @@ public class DateTimeFormatters {
           .toFormatter(Locale.ROOT)
           .withResolverStyle(ResolverStyle.STRICT);
 
+  // MDD
+  public static final DateTimeFormatter DATE_FORMATTER_SINGLE_DIGIT_MONTH =
+      new DateTimeFormatterBuilder()
+          .parseDefaulting(YEAR, 2000)
+          .appendPattern("Mdd")
+          .toFormatter()
+          .withResolverStyle(ResolverStyle.STRICT);
+
+  // MMDD
+  public static final DateTimeFormatter DATE_FORMATTER_NO_YEAR =
+      new DateTimeFormatterBuilder()
+          .parseDefaulting(YEAR, 2000)
+          .appendPattern("MMdd")
+          .toFormatter()
+          .withResolverStyle(ResolverStyle.STRICT);
+
+  // YMMDD
+  public static final DateTimeFormatter DATE_FORMATTER_SINGLE_DIGIT_YEAR =
+      new DateTimeFormatterBuilder()
+          .appendValueReduced(YEAR, 1, 1, 2000)
+          .appendPattern("MMdd")
+          .toFormatter()
+          .withResolverStyle(ResolverStyle.STRICT);
+
   // YYMMDD
   public static final DateTimeFormatter DATE_FORMATTER_SHORT_YEAR =
       new DateTimeFormatterBuilder()

diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ToSecondsTest.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.expression.datetime;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.opensearch.sql.data.type.ExprCoreType.LONG;
+import static org.opensearch.sql.expression.datetime.DateTimeFunction.SECONDS_PER_DAY;
+
+import java.time.Duration;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.opensearch.sql.data.model.ExprDateValue;
+import org.opensearch.sql.data.model.ExprDatetimeValue;
+import org.opensearch.sql.data.model.ExprIntervalValue;
+import org.opensearch.sql.data.model.ExprLongValue;
+import org.opensearch.sql.data.model.ExprNullValue;
+import org.opensearch.sql.data.model.ExprStringValue;
+import org.opensearch.sql.data.model.ExprTimeValue;
+import org.opensearch.sql.data.model.ExprTimestampValue;
+import org.opensearch.sql.data.model.ExprValue;
+import org.opensearch.sql.exception.SemanticCheckException;
+import org.opensearch.sql.expression.DSL;
+import org.opensearch.sql.expression.Expression;
+import org.opensearch.sql.expression.ExpressionTestBase;
+import org.opensearch.sql.expression.FunctionExpression;
+
+
+class ToSecondsTest extends ExpressionTestBase {
+
+  private static final long SECONDS_FROM_0001_01_01_TO_EPOCH_START = 62167219200L;
+
+  private static Stream<Arguments> getTestDataForToSeconds() {
+    return Stream.of(
+        Arguments.of(new ExprLongValue(101), new ExprLongValue(63113904000L)),
+        Arguments.of(new ExprLongValue(1030), new ExprLongValue(63140083200L)),
+        Arguments.of(new ExprLongValue(50101), new ExprLongValue(63271756800L)),
+        Arguments.of(new ExprLongValue(950501), new ExprLongValue(62966505600L)),
+        Arguments.of(new ExprLongValue(19950501), new ExprLongValue(62966505600L)),
+        Arguments.of(new ExprLongValue(9950501), ExprNullValue.of()),
+        Arguments.of(new ExprLongValue(-123L), ExprNullValue.of()),
+        Arguments.of(new ExprLongValue(1), ExprNullValue.of()),
+        Arguments.of(new ExprLongValue(919950501), ExprNullValue.of()),
+        Arguments.of(new ExprStringValue("2009-11-29 00:00:00"), new ExprLongValue(63426672000L)),
+        Arguments.of(new ExprStringValue("2009-11-29 13:43:32"), new ExprLongValue(63426721412L)),
+        Arguments.of(new ExprDateValue("2009-11-29"), new ExprLongValue(63426672000L)),
+        Arguments.of(new ExprDatetimeValue("2009-11-29 13:43:32"),
+            new ExprLongValue(63426721412L)),
+        Arguments.of(new ExprTimestampValue("2009-11-29 13:43:32"),
+            new ExprLongValue(63426721412L))
+    );
+  }
+
+  @ParameterizedTest
+  @MethodSource("getTestDataForToSeconds")
+  public void testToSeconds(ExprValue arg, ExprValue expected) {
+    FunctionExpression expr = DSL.to_seconds(DSL.literal(arg));
+    assertEquals(LONG, expr.type());
+    assertEquals(expected, eval(expr));
+  }
+
+  @Test
+  public void testToSecondsWithTimeType() {
+    FunctionExpression expr = DSL.to_seconds(functionProperties,
+        DSL.literal(new ExprTimeValue("10:11:12")));
+
+    long expected = SECONDS_FROM_0001_01_01_TO_EPOCH_START
+        + LocalDate.now(functionProperties.getQueryStartClock())
+            .toEpochSecond(LocalTime.parse("10:11:12"), ZoneOffset.UTC);
+
+    assertEquals(expected, eval(expr).longValue());
+  }
+
+  private static Stream<Arguments> getInvalidTestDataForToSeconds() {
+    return Stream.of(
+        Arguments.of(new ExprStringValue("asdfasdf")),
+        Arguments.of(new ExprStringValue("2000-14-10")),
+        Arguments.of(new ExprStringValue("2000-10-45")),
+        Arguments.of(new ExprStringValue("2000-10-10 70:00:00")),
+        Arguments.of(new ExprStringValue("2000-10-10 00:70:00")),
+        Arguments.of(new ExprStringValue("2000-10-10 00:00:70"))
+    );
+  }
+
+  @ParameterizedTest
+  @MethodSource("getInvalidTestDataForToSeconds")
+  public void testToSecondsInvalidArg(ExprValue arg) {
+    FunctionExpression expr = DSL.to_seconds(DSL.literal(arg));
+    assertThrows(SemanticCheckException.class, () -> eval(expr));
+  }
+
+  @Test
+  public void testToSecondsWithDateAdd() {
+    LocalDate date = LocalDate.of(2000, 1, 1);
+    FunctionExpression dateExpr = DSL.to_seconds(DSL.literal(new ExprDateValue(date)));
+    long addedSeconds = SECONDS_PER_DAY;
+    long expected = eval(dateExpr).longValue() + addedSeconds;
+
+    FunctionExpression dateAddExpr = DSL.date_add(
+        DSL.literal(new ExprDateValue(date)),
+        DSL.literal(new ExprIntervalValue(Duration.ofSeconds(addedSeconds))));
+
+    long result = eval(DSL.to_seconds(DSL.literal(eval(dateAddExpr)))).longValue();
+
+    assertEquals(expected, result);
+  }
+
+  private ExprValue eval(Expression expression) {
+    return expression.valueOf();
+  }
+}
diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst
@@ -2619,6 +2619,28 @@ Example::
     | 733687                       |
     +------------------------------+
 
+TO_SECONDS
+----------
+
+Description
+>>>>>>>>>>>
+
+Usage: to_seconds(date) returns the number of seconds since the year 0 of the given value. Returns NULL if value is invalid.
+An argument of a LONG type can be used. It must be formatted as YMMDD, YYMMDD, YYYMMDD or YYYYMMDD. Note that a LONG type argument cannot have leading 0s as it will be parsed using an octal numbering system.
+
+Argument type: STRING/LONG/DATE/DATETIME/TIME/TIMESTAMP
+
+Return type: LONG
+
+Example::
+
+    os> SELECT TO_SECONDS(DATE '2008-10-07'), TO_SECONDS(950228)
+    fetched rows / total rows = 1/1
+    +---------------------------------+----------------------+
+    | TO_SECONDS(DATE '2008-10-07')   | TO_SECONDS(950228)   |
+    |---------------------------------+----------------------|
+    | 63390556800                     | 62961148800          |
+    +---------------------------------+----------------------+
 
 UNIX_TIMESTAMP
 --------------