[ML] Add support for date_nanos fields in find_file_structure

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinder.java

@@ -149,14 +149,15 @@ static DelimitedFileStructureFinder makeDelimitedFileStructureFinder(List<String
                 .setJavaTimestampFormats(timeField.v2().getJavaTimestampFormats())
                 .setNeedClientTimezone(needClientTimeZone)
                 .setIngestPipeline(FileStructureUtils.makeIngestPipelineDefinition(null, Collections.emptyMap(), csvProcessorSettings,
-                    mappings, timeField.v1(), timeField.v2().getJavaTimestampFormats(), needClientTimeZone))
+                    mappings, timeField.v1(), timeField.v2().getJavaTimestampFormats(), needClientTimeZone,
+                    timeField.v2().needNanosecondPrecision()))
                 .setMultilineStartPattern(makeMultilineStartPattern(explanation, columnNamesList, maxLinesPerMessage, delimiterPattern,
                     quotePattern, mappings, timeField.v1(), timeField.v2()));
 
-            mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT);
+            mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, timeField.v2().getEsDateMappingTypeWithoutFormat());
         } else {
             structureBuilder.setIngestPipeline(FileStructureUtils.makeIngestPipelineDefinition(null, Collections.emptyMap(),
-                csvProcessorSettings, mappings, null, null, false));
+                csvProcessorSettings, mappings, null, null, false, false));
             structureBuilder.setMultilineStartPattern(makeMultilineStartPattern(explanation, columnNamesList, maxLinesPerMessage,
                 delimiterPattern, quotePattern, mappings, null, null));
         }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtils.java

@@ -36,6 +36,7 @@ public final class FileStructureUtils {
     public static final String MAPPING_PROPERTIES_SETTING = "properties";
     public static final Map<String, String> DATE_MAPPING_WITHOUT_FORMAT =
         Collections.singletonMap(MAPPING_TYPE_SETTING, "date");
+    public static final String NANOSECOND_DATE_OUTPUT_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSXXX";
     public static final Set<String> CONVERTIBLE_TYPES =
         Collections.unmodifiableSet(Sets.newHashSet("integer", "long", "float", "double", "boolean"));
 
@@ -397,13 +398,15 @@ static boolean isMoreLikelyTextThanKeyword(String str) {
      * @param timestampFormats Timestamp formats to be used for parsing {@code timestampField}.
      *                         May be <code>null</code> if {@code timestampField} is also <code>null</code>.
      * @param needClientTimezone Is the timezone of the client supplying data to ingest required to uniquely parse the timestamp?
+     * @param needNanosecondPrecision Does the timestamp have more than millisecond accuracy?
      * @return The ingest pipeline definition, or <code>null</code> if none is required.
      */
     public static Map<String, Object> makeIngestPipelineDefinition(String grokPattern, Map<String, String> customGrokPatternDefinitions,
                                                                    Map<String, Object> csvProcessorSettings,
                                                                    Map<String, Object> mappingsForConversions,
                                                                    String timestampField, List<String> timestampFormats,
-                                                                   boolean needClientTimezone) {
+                                                                   boolean needClientTimezone,
+                                                                   boolean needNanosecondPrecision) {
 
         if (grokPattern == null && csvProcessorSettings == null && timestampField == null) {
             return null;
@@ -437,6 +440,9 @@ public static Map<String, Object> makeIngestPipelineDefinition(String grokPatter
                 dateProcessorSettings.put("timezone", "{{ " + BEAT_TIMEZONE_FIELD + " }}");
             }
             dateProcessorSettings.put("formats", timestampFormats);
+            if (needNanosecondPrecision) {
+                dateProcessorSettings.put("output_format", NANOSECOND_DATE_OUTPUT_FORMAT);
+            }
             processors.add(Collections.singletonMap("date", dateProcessorSettings));
         }
 

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinder.java

@@ -64,15 +64,16 @@ static NdJsonFileStructureFinder makeNdJsonFileStructureFinder(List<String> expl
                 .setIngestPipeline(FileStructureUtils.makeIngestPipelineDefinition(null, Collections.emptyMap(), null,
                     // Note: no convert processors are added based on mappings for NDJSON input
                     // because it's reasonable that _source matches the supplied JSON precisely
-                    Collections.emptyMap(), timeField.v1(), timeField.v2().getJavaTimestampFormats(), needClientTimeZone));
+                    Collections.emptyMap(), timeField.v1(), timeField.v2().getJavaTimestampFormats(), needClientTimeZone,
+                    timeField.v2().needNanosecondPrecision()));
         }
 
         Tuple<SortedMap<String, Object>, SortedMap<String, FieldStats>> mappingsAndFieldStats =
             FileStructureUtils.guessMappingsAndCalculateFieldStats(explanation, sampleRecords, timeoutChecker);
 
-        SortedMap<String, Object> mappings = mappingsAndFieldStats.v1();
+        Map<String, Object> mappings = mappingsAndFieldStats.v1();
         if (timeField != null) {
-            mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT);
+            mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, timeField.v2().getEsDateMappingTypeWithoutFormat());
         }
 
         if (mappingsAndFieldStats.v2() != null) {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinder.java

@@ -111,7 +111,7 @@ static TextLogFileStructureFinder makeTextLogFileStructureFinder(List<String> ex
         Map<String, String> messageMapping = Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "text");
         SortedMap<String, Object> mappings = new TreeMap<>();
         mappings.put("message", messageMapping);
-        mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT);
+        mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, timestampFormatFinder.getEsDateMappingTypeWithoutFormat());
 
         SortedMap<String, FieldStats> fieldStats = new TreeMap<>();
         fieldStats.put("message", FileStructureUtils.calculateFieldStats(messageMapping, sampleMessages, timeoutChecker));
@@ -151,7 +151,8 @@ static TextLogFileStructureFinder makeTextLogFileStructureFinder(List<String> ex
             .setNeedClientTimezone(needClientTimeZone)
             .setGrokPattern(grokPattern)
             .setIngestPipeline(FileStructureUtils.makeIngestPipelineDefinition(grokPattern, customGrokPatternDefinitions, null, mappings,
-                interimTimestampField, timestampFormatFinder.getJavaTimestampFormats(), needClientTimeZone))
+                interimTimestampField, timestampFormatFinder.getJavaTimestampFormats(), needClientTimeZone,
+                timestampFormatFinder.needNanosecondPrecision()))
             .setMappings(mappings)
             .setFieldStats(fieldStats)
             .setExplanation(explanation)

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java

@@ -53,6 +53,8 @@ public final class TimestampFormatFinder {
     private static final Logger logger = LogManager.getLogger(TimestampFormatFinder.class);
     private static final String PUNCTUATION_THAT_NEEDS_ESCAPING_IN_REGEX = "\\|()[]{}^$.*?";
     private static final String FRACTIONAL_SECOND_SEPARATORS = ":.,";
+    private static final Pattern FRACTIONAL_SECOND_INTERPRETER =
+        Pattern.compile("([" + FRACTIONAL_SECOND_SEPARATORS + "])(\\d{3,9})($|[Z+-])");
     private static final char INDETERMINATE_FIELD_PLACEHOLDER = '?';
     // The ? characters in this must match INDETERMINATE_FIELD_PLACEHOLDER
     // above, but they're literals in this regex to aid readability
@@ -702,6 +704,20 @@ public List<String> getJavaTimestampFormats() {
             (matchedFormats.size() > 1) ? matchedFormats.get(0) : null);
     }
 
+    /**
+     * This is needed to decide between "date" and "date_nanos" as the index mapping type.
+     * @return Do the observed timestamps require nanosecond precision to store accurately?
+     */
+    public boolean needNanosecondPrecision() {
+        if (matchedFormats.isEmpty()) {
+            // If errorOnNoTimestamp is set and we get here it means no samples have been added, which is likely a programmer mistake
+            assert errorOnNoTimestamp == false;
+            return false;
+        }
+        return matches.stream().filter(match -> matchedFormats.size() < 2 || matchedFormats.get(0).canMergeWith(match.timestampFormat))
+            .anyMatch(match -> match.hasNanosecondPrecision);
+    }
+
     /**
      * Given a list of timestamp formats that might contain indeterminate day/month parts,
      * return the corresponding pattern with the placeholders replaced with concrete
@@ -947,6 +963,14 @@ public boolean hasTimezoneDependentParsing() {
             .anyMatch(match -> match.hasTimezoneDependentParsing);
     }
 
+    /**
+     * The @timestamp field will always have been parsed into epoch format,
+     * so we just need to know if it has nanosecond resolution or not.
+     */
+    public Map<String, String> getEsDateMappingTypeWithoutFormat() {
+        return Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, needNanosecondPrecision() ? "date_nanos" : "date");
+    }
+
     /**
      * Sometimes Elasticsearch mappings for dates need to include the format.
      * This method returns appropriate mappings settings: at minimum "type" : "date",
@@ -959,7 +983,7 @@ public Map<String, String> getEsDateMappingTypeWithFormat() {
             return Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "keyword");
         }
         Map<String, String> mapping = new LinkedHashMap<>();
-        mapping.put(FileStructureUtils.MAPPING_TYPE_SETTING, "date");
+        mapping.put(FileStructureUtils.MAPPING_TYPE_SETTING, needNanosecondPrecision() ? "date_nanos" : "date");
         String formats = javaTimestampFormats.stream().map(format -> {
             switch (format) {
                 case "ISO8601":
@@ -1233,6 +1257,7 @@ static final class TimestampMatch {
         final int secondIndeterminateDateNumber;
 
         final boolean hasTimezoneDependentParsing;
+        final boolean hasNanosecondPrecision;
 
         /**
          * Text that came after the timestamp in the matched field/message.
@@ -1250,6 +1275,8 @@ static final class TimestampMatch {
             this.secondIndeterminateDateNumber = indeterminateDateNumbers[1];
             this.hasTimezoneDependentParsing = requiresTimezoneDependentParsing(timestampFormat.rawJavaTimestampFormats.get(0),
                 matchedDate);
+            this.hasNanosecondPrecision = matchHasNanosecondPrecision(timestampFormat.rawJavaTimestampFormats.get(0),
+                matchedDate);
             this.epilogue = Objects.requireNonNull(epilogue);
         }
 
@@ -1259,6 +1286,7 @@ static final class TimestampMatch {
             this.firstIndeterminateDateNumber = toCopyExceptFormat.firstIndeterminateDateNumber;
             this.secondIndeterminateDateNumber = toCopyExceptFormat.secondIndeterminateDateNumber;
             this.hasTimezoneDependentParsing = toCopyExceptFormat.hasTimezoneDependentParsing;
+            this.hasNanosecondPrecision = toCopyExceptFormat.hasNanosecondPrecision;
             this.epilogue = toCopyExceptFormat.epilogue;
         }
 
@@ -1285,6 +1313,43 @@ static boolean requiresTimezoneDependentParsing(String format, String matchedDat
             }
         }
 
+        static boolean matchHasNanosecondPrecision(String format, String matchedDate) {
+            switch (format) {
+                case "ISO8601":
+                    Matcher matcher = FRACTIONAL_SECOND_INTERPRETER.matcher(matchedDate);
+                    return matcher.find() && matcher.group(2).length() > 3;
+                case "UNIX_MS":
+                case "UNIX":
+                    return false;
+                case "TAI64N":
+                    return true;
+                default:
+                    boolean notQuoted = true;
+                    int consecutiveSs = 0;
+                    for (int pos = 0; pos < format.length(); ++pos) {
+                        char curChar = format.charAt(pos);
+                        if (curChar == '\'') {
+                            // Literal single quotes are escaped by using two consecutive single quotes.
+                            // Technically this code does the wrong thing in this case, as it flips quoting
+                            // from off to on or on to off and then back.  However, since by definition there
+                            // is nothing in between the consecutive single quotes in this case, the net
+                            // effect is correct and good enough for what this method is doing.
+                            notQuoted = !notQuoted;
+                            consecutiveSs = 0;
+                        } else if (notQuoted) {
+                            if (curChar == 'S') {
+                                if (++consecutiveSs > 3) {
+                                    return true;
+                                }
+                            } else {
+                                consecutiveSs = 0;
+                            }
+                        }
+                    }
+                    return false;
+            }
+        }
+
         static int[] parseIndeterminateDateNumbers(String matchedDate, List<String> rawJavaTimestampFormats) {
             int[] indeterminateDateNumbers = { -1, -1 };
 
@@ -1368,7 +1433,6 @@ public String toString() {
      */
     static final class CandidateTimestampFormat {
 
-        private static final Pattern FRACTIONAL_SECOND_INTERPRETER = Pattern.compile("([" + FRACTIONAL_SECOND_SEPARATORS + "])(\\d{3,9})$");
         // This means that in the case of a literal Z, XXX is preferred
         private static final Pattern TRAILING_OFFSET_WITHOUT_COLON_FINDER = Pattern.compile("[+-]\\d{4}$");
 

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinder.java

@@ -104,7 +104,7 @@ static XmlFileStructureFinder makeXmlFileStructureFinder(List<String> explanatio
                 .setNeedClientTimezone(needClientTimeZone)
                 .setIngestPipeline(FileStructureUtils.makeIngestPipelineDefinition(null, Collections.emptyMap(), null,
                     Collections.emptyMap(), topLevelTag + "." + timeField.v1(), timeField.v2().getJavaTimestampFormats(),
-                    needClientTimeZone));
+                    needClientTimeZone, timeField.v2().needNanosecondPrecision()));
         }
 
         Tuple<SortedMap<String, Object>, SortedMap<String, FieldStats>> mappingsAndFieldStats =
@@ -114,14 +114,14 @@ static XmlFileStructureFinder makeXmlFileStructureFinder(List<String> explanatio
             structureBuilder.setFieldStats(mappingsAndFieldStats.v2());
         }
 
-        SortedMap<String, Object> innerMappings = mappingsAndFieldStats.v1();
+        Map<String, Object> innerMappings = mappingsAndFieldStats.v1();
         Map<String, Object> secondLevelProperties = new LinkedHashMap<>();
         secondLevelProperties.put(FileStructureUtils.MAPPING_TYPE_SETTING, "object");
         secondLevelProperties.put(FileStructureUtils.MAPPING_PROPERTIES_SETTING, innerMappings);
         SortedMap<String, Object> outerMappings = new TreeMap<>();
         outerMappings.put(topLevelTag, secondLevelProperties);
         if (timeField != null) {
-            outerMappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT);
+            outerMappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, timeField.v2().getEsDateMappingTypeWithoutFormat());
         }
 
         FileStructure structure = structureBuilder