[IOTDB-4156] import csv has error time after timestamp_precision set ns (apache#7032)

cmlmakahts · web-flow · commit 81f5ca61c290 · 2022-08-18T18:24:41.000+08:00
diff --git a/cli/pom.xml b/cli/pom.xml
@@ -83,7 +83,12 @@
             <groupId>org.apache.iotdb</groupId>
             <artifactId>iotdb-server</artifactId>
             <version>${project.version}</version>
-            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
@@ -124,6 +129,36 @@
                     <skipITs>${cli.it.skip}</skipITs>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.3.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
+                            <artifactSet>
+                                <includes>
+                                    <include>org.apache.iotdb:iotdb-cli</include>
+                                    <include>org.apache.iotdb:iotdb-server</include>
+                                </includes>
+                            </artifactSet>
+                            <filters>
+                                <filter>
+                                    <artifact>org.apache.iotdb:iotdb-server</artifact>
+                                    <includes>
+                                        <include>org/apache/iotdb/db/qp/utils/*</include>
+                                    </includes>
+                                </filter>
+                            </filters>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-assembly-plugin</artifactId>
diff --git a/cli/src/assembly/cli.xml b/cli/src/assembly/cli.xml
@@ -28,6 +28,9 @@
     <includeBaseDirectory>false</includeBaseDirectory>
     <dependencySets>
         <dependencySet>
+            <excludes>
+                <exclude>org.apache.iotdb:iotdb-server</exclude>
+            </excludes>
             <outputDirectory>lib</outputDirectory>
         </dependencySet>
     </dependencySets>
diff --git a/cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java b/cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
@@ -19,6 +19,7 @@
 
 package org.apache.iotdb.tool;
 
+import org.apache.iotdb.db.qp.utils.DatetimeUtils;
 import org.apache.iotdb.exception.ArgsErrorException;
 import org.apache.iotdb.rpc.IoTDBConnectionException;
 import org.apache.iotdb.rpc.StatementExecutionException;
@@ -42,7 +43,6 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -82,6 +82,9 @@ public class ImportCsv extends AbstractCsvTool {
   private static final String CSV_SUFFIXS = "csv";
   private static final String TXT_SUFFIXS = "txt";
 
+  private static final String TIMESTAMP_PRECISION_ARGS = "tp";
+  private static final String TIMESTAMP_PRECISION_NAME = "timestamp precision (ms/us/ns)";
+
   private static final String TSFILEDB_CLI_PREFIX = "ImportCsv";
 
   private static String targetPath;
@@ -93,6 +96,8 @@ public class ImportCsv extends AbstractCsvTool {
 
   private static int batchPointSize = 100_000;
 
+  private static String timestampPrecision = "ms";
+
   /**
    * create the commandline options.
    *
@@ -153,6 +158,14 @@ private static Options createOptions() {
             .build();
     options.addOption(opBatchPointSize);
 
+    Option opTimestampPrecision =
+        Option.builder(TIMESTAMP_PRECISION_ARGS)
+            .argName(TIMESTAMP_PRECISION_ARGS)
+            .hasArg()
+            .desc("Timestamp precision (ms/us/ns)")
+            .build();
+    options.addOption(opTimestampPrecision);
+
     return options;
   }
 
@@ -178,6 +191,10 @@ private static void parseSpecialParams(CommandLine commandLine) {
     if (commandLine.getOptionValue(ALIGNED_ARGS) != null) {
       aligned = Boolean.valueOf(commandLine.getOptionValue(ALIGNED_ARGS));
     }
+
+    if (commandLine.getOptionValue(TIMESTAMP_PRECISION_ARGS) != null) {
+      timestampPrecision = commandLine.getOptionValue(TIMESTAMP_PRECISION_ARGS);
+    }
   }
 
   public static void main(String[] args) throws IoTDBConnectionException {
@@ -345,7 +362,6 @@ private static void writeDataAlignedByTime(
     List<List<TSDataType>> typesList = new ArrayList<>();
     List<List<Object>> valuesList = new ArrayList<>();
 
-    AtomicReference<SimpleDateFormat> timeFormatter = new AtomicReference<>(null);
     AtomicReference<Boolean> hasStarted = new AtomicReference<>(false);
     AtomicInteger pointSize = new AtomicInteger(0);
 
@@ -355,7 +371,6 @@ private static void writeDataAlignedByTime(
         record -> {
           if (!hasStarted.get()) {
             hasStarted.set(true);
-            timeFormatter.set(formatterInit(record.get(0)));
           } else if (pointSize.get() >= batchPointSize) {
             writeAndEmptyDataSet(deviceIds, times, typesList, valuesList, measurementsList, 3);
             pointSize.set(0);
@@ -403,17 +418,7 @@ record -> {
               }
             }
             if (!measurements.isEmpty()) {
-              if (timeFormatter.get() == null) {
-                times.add(Long.valueOf(record.get(timeColumn)));
-              } else {
-                try {
-                  times.add(timeFormatter.get().parse(record.get(timeColumn)).getTime());
-                } catch (ParseException e) {
-                  System.out.println(
-                      "Meet error when insert csv because the format of time is not supported");
-                  System.exit(0);
-                }
-              }
+              times.add(parseTimestamp(record.get(timeColumn)));
               deviceIds.add(deviceId);
               typesList.add(types);
               valuesList.add(values);
@@ -472,7 +477,7 @@ record -> {
           // only run in first record
           if (deviceName.get() == null) {
             deviceName.set(record.get(1));
-            timeFormatter.set(formatterInit(record.get(0)));
+            // timeFormatter.set(formatterInit(record.get(0)));
           } else if (!Objects.equals(deviceName.get(), record.get(1))) {
             // if device changed
             writeAndEmptyDataSet(
@@ -547,13 +552,7 @@ record -> {
             if (timeFormatter.get() == null) {
               times.add(Long.valueOf(record.get(timeColumn)));
             } else {
-              try {
-                times.add(timeFormatter.get().parse(record.get(timeColumn)).getTime());
-              } catch (ParseException e) {
-                System.out.println(
-                    "Meet error when insert csv because the format of time is not supported");
-                System.exit(0);
-              }
+              times.add(parseTimestamp(record.get(timeColumn)));
             }
             typesList.add(types);
             valuesList.add(values);
@@ -739,32 +738,6 @@ private static boolean queryType(
     }
   }
 
-  /**
-   * return a suit time formatter
-   *
-   * @param time
-   * @return
-   */
-  private static SimpleDateFormat formatterInit(String time) {
-    try {
-      Long.parseLong(time);
-      return null;
-    } catch (Exception ignored) {
-      // do nothing
-    }
-
-    for (String timeFormat : STRING_TIME_FORMAT) {
-      SimpleDateFormat format = new SimpleDateFormat(timeFormat);
-      try {
-        format.parse(time);
-        return format;
-      } catch (java.text.ParseException ignored) {
-        // do nothing
-      }
-    }
-    return null;
-  }
-
   /**
    * return the TSDataType
    *
@@ -854,4 +827,14 @@ private static Object typeTrans(String value, TSDataType type) {
       return null;
     }
   }
+
+  private static long parseTimestamp(String str) {
+    long timestamp;
+    try {
+      timestamp = Long.parseLong(str);
+    } catch (NumberFormatException e) {
+      timestamp = DatetimeUtils.convertDatetimeStrToLong(str, zoneId, timestampPrecision);
+    }
+    return timestamp;
+  }
 }
diff --git a/docs/UserGuide/Write-And-Delete-Data/CSV-Tool.md b/docs/UserGuide/Write-And-Delete-Data/CSV-Tool.md
@@ -174,9 +174,9 @@ Time,Device,str(TEXT),int(INT32)
 
 ```shell
 # Unix/OS X
-> tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f <xxx.csv> [-fd <./failedDirectory>] [-aligned <true>]
+> tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f <xxx.csv> [-fd <./failedDirectory>] [-aligned <true>] [-tp <ms/ns/us>]
 # Windows
-> tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f <xxx.csv> [-fd <./failedDirectory>] [-aligned <true>]
+> tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f <xxx.csv> [-fd <./failedDirectory>] [-aligned <true>] [-tp <ms/ns/us>]
 ```
 
 Description:
@@ -197,6 +197,9 @@ Description:
   - specifying the point's number of a batch. If the program throw the exception `org.apache.thrift.transport.TTransportException: Frame size larger than protect max size`, you can lower this parameter as appropriate.
   - example: `-batch 100000`, `100000` is the default value.
 
+* `-tp <time-precision>`:
+  - specifying a time precision. Options includes `ms`(millisecond), `ns`(nanosecond), and `us`(microsecond), `ms` is default.
+
 ### Example
 
 ```sh
@@ -208,6 +211,8 @@ Description:
 > tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f example-filename.csv
 # or
 > tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f example-filename.csv -fd .\failed
+# or
+> tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f example-filename.csv -fd .\failed -tp ns
 ```
 
 ### Note
diff --git a/docs/zh/UserGuide/Write-And-Delete-Data/CSV-Tool.md b/docs/zh/UserGuide/Write-And-Delete-Data/CSV-Tool.md
@@ -175,9 +175,9 @@ Time,Device,str(TEXT),int(INT32)
 
 ```shell
 # Unix/OS X
->tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f <xxx.csv> [-fd <./failedDirectory>] [-aligned <true>]
+>tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f <xxx.csv> [-fd <./failedDirectory>] [-aligned <true>] [-tp <ms/ns/us>]
 # Windows
->tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f <xxx.csv> [-fd <./failedDirectory>] [-aligned <true>]
+>tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f <xxx.csv> [-fd <./failedDirectory>] [-aligned <true>] [-tp <ms/ns/us>]
 ```
 
 参数:
@@ -198,6 +198,9 @@ Time,Device,str(TEXT),int(INT32)
   - 用于指定每一批插入的数据的点数。如果程序报了`org.apache.thrift.transport.TTransportException: Frame size larger than protect max size`这个错的话，就可以适当的调低这个参数。
   - 例如: `-batch 100000`，`100000`是默认值。
 
+* `-tp`:
+  - 用于指定时间精度，可选值包括`ms`（毫秒），`ns`（纳秒），`us`（微秒），默认值为`ms`。
+
 ### 运行示例
 
 ```sh
@@ -209,6 +212,8 @@ Time,Device,str(TEXT),int(INT32)
 >tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f example-filename.csv
 # or
 >tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f example-filename.csv -fd .\failed
+# or
+> tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f example-filename.csv -fd .\failed -tp ns
 ```
 
 ### 注意
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/utils/DatetimeUtils.java b/server/src/main/java/org/apache/iotdb/db/qp/utils/DatetimeUtils.java
@@ -450,7 +450,16 @@ private DatetimeUtils() {
           .toFormatter();
 
   public static long convertDatetimeStrToLong(String str, ZoneId zoneId) {
-    return convertDatetimeStrToLong(str, toZoneOffset(zoneId), 0);
+    return convertDatetimeStrToLong(
+        str,
+        toZoneOffset(zoneId),
+        0,
+        IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision());
+  }
+
+  public static long convertDatetimeStrToLong(
+      String str, ZoneId zoneId, String timestampPrecision) {
+    return convertDatetimeStrToLong(str, toZoneOffset(zoneId), 0, timestampPrecision);
   }
 
   public static long getInstantWithPrecision(String str, String timestampPrecision) {
@@ -478,10 +487,8 @@ public static long getInstantWithPrecision(String str, String timestampPrecision
   }
 
   /** convert date time string to millisecond, microsecond or nanosecond. */
-  public static long convertDatetimeStrToLong(String str, ZoneOffset offset, int depth) {
-
-    String timestampPrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
-
+  public static long convertDatetimeStrToLong(
+      String str, ZoneOffset offset, int depth, String timestampPrecision) {
     if (depth >= 2) {
       throw new DateTimeException(
           String.format(
@@ -490,12 +497,13 @@ public static long convertDatetimeStrToLong(String str, ZoneOffset offset, int d
               str, offset));
     }
     if (str.contains("Z")) {
-      return convertDatetimeStrToLong(str.substring(0, str.indexOf('Z')) + "+00:00", offset, depth);
+      return convertDatetimeStrToLong(
+          str.substring(0, str.indexOf('Z')) + "+00:00", offset, depth, timestampPrecision);
     } else if (str.length() == 10) {
-      return convertDatetimeStrToLong(str + "T00:00:00", offset, depth);
+      return convertDatetimeStrToLong(str + "T00:00:00", offset, depth, timestampPrecision);
     } else if (str.length() - str.lastIndexOf('+') != 6
         && str.length() - str.lastIndexOf('-') != 6) {
-      return convertDatetimeStrToLong(str + offset, offset, depth + 1);
+      return convertDatetimeStrToLong(str + offset, offset, depth + 1, timestampPrecision);
     } else if (str.contains("[") || str.contains("]")) {
       throw new DateTimeException(
           String.format(
diff --git a/server/src/test/java/org/apache/iotdb/db/qp/utils/DatetimeQueryDataSetUtilsTest.java b/server/src/test/java/org/apache/iotdb/db/qp/utils/DatetimeQueryDataSetUtilsTest.java
@@ -189,7 +189,7 @@ public void testConvertDatetimeStrToLongWithoutMS(ZoneOffset zoneOffset, ZoneId
           "2019.01.02T15:13:27" + zoneOffset,
         };
     for (String str : timeFormatWithoutMs) {
-      Assert.assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str, zoneOffset, 0));
+      Assert.assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str, zoneOffset, 0, "ms"));
     }
 
     for (String str : timeFormatWithoutMs) {
@@ -215,7 +215,7 @@ public void testConvertDatetimeStrToLongWithMS(ZoneOffset zoneOffset, ZoneId zon
           "2019.01.02T15:13:27.689" + zoneOffset,
         };
     for (String str : timeFormatWithoutMs) {
-      assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str, zoneOffset, 0));
+      assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str, zoneOffset, 0, "ms"));
     }
 
     for (String str : timeFormatWithoutMs) {
@@ -230,7 +230,7 @@ public void testConvertDateStrToLong(ZoneOffset zoneOffset, ZoneId zoneId, long
           "2019-01-02", "2019/01/02", "2019.01.02",
         };
     for (String str : timeFormatWithoutMs) {
-      assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str, zoneOffset, 0));
+      assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str, zoneOffset, 0, "ms"));
     }
 
     for (String str : timeFormatWithoutMs) {
