[7.x][ML] Data frame analytics analysis stats (#53788)

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java

@@ -20,12 +20,15 @@
 package org.elasticsearch.client.ml.dataframe;
 
 import org.elasticsearch.client.ml.NodeAttributes;
+import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.client.ml.dataframe.stats.common.MemoryUsage;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.inject.internal.ToStringBuilder;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
 
 import java.io.IOException;
 import java.util.List;
@@ -45,6 +48,7 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
     static final ParseField FAILURE_REASON = new ParseField("failure_reason");
     static final ParseField PROGRESS = new ParseField("progress");
     static final ParseField MEMORY_USAGE = new ParseField("memory_usage");
+    static final ParseField ANALYSIS_STATS = new ParseField("analysis_stats");
     static final ParseField NODE = new ParseField("node");
     static final ParseField ASSIGNMENT_EXPLANATION = new ParseField("assignment_explanation");
 
@@ -57,8 +61,9 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
                 (String) args[2],
                 (List<PhaseProgress>) args[3],
                 (MemoryUsage) args[4],
-                (NodeAttributes) args[5],
-                (String) args[6]));
+                (AnalysisStats) args[5],
+                (NodeAttributes) args[6],
+                (String) args[7]));
 
     static {
         PARSER.declareString(constructorArg(), ID);
@@ -71,26 +76,38 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
         PARSER.declareString(optionalConstructorArg(), FAILURE_REASON);
         PARSER.declareObjectArray(optionalConstructorArg(), PhaseProgress.PARSER, PROGRESS);
         PARSER.declareObject(optionalConstructorArg(), MemoryUsage.PARSER, MEMORY_USAGE);
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseAnalysisStats(p), ANALYSIS_STATS);
         PARSER.declareObject(optionalConstructorArg(), NodeAttributes.PARSER, NODE);
         PARSER.declareString(optionalConstructorArg(), ASSIGNMENT_EXPLANATION);
     }
 
+    private static AnalysisStats parseAnalysisStats(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+        AnalysisStats analysisStats = parser.namedObject(AnalysisStats.class, parser.currentName(), true);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        return analysisStats;
+    }
+
     private final String id;
     private final DataFrameAnalyticsState state;
     private final String failureReason;
     private final List<PhaseProgress> progress;
     private final MemoryUsage memoryUsage;
+    private final AnalysisStats analysisStats;
     private final NodeAttributes node;
     private final String assignmentExplanation;
 
     public DataFrameAnalyticsStats(String id, DataFrameAnalyticsState state, @Nullable String failureReason,
                                    @Nullable List<PhaseProgress> progress, @Nullable MemoryUsage memoryUsage,
-                                   @Nullable NodeAttributes node, @Nullable String assignmentExplanation) {
+                                   @Nullable AnalysisStats analysisStats, @Nullable NodeAttributes node,
+                                   @Nullable String assignmentExplanation) {
         this.id = id;
         this.state = state;
         this.failureReason = failureReason;
         this.progress = progress;
         this.memoryUsage = memoryUsage;
+        this.analysisStats = analysisStats;
         this.node = node;
         this.assignmentExplanation = assignmentExplanation;
     }
@@ -116,6 +133,11 @@ public MemoryUsage getMemoryUsage() {
         return memoryUsage;
     }
 
+    @Nullable
+    public AnalysisStats getAnalysisStats() {
+        return analysisStats;
+    }
+
     public NodeAttributes getNode() {
         return node;
     }
@@ -135,13 +157,14 @@ public boolean equals(Object o) {
             && Objects.equals(failureReason, other.failureReason)
             && Objects.equals(progress, other.progress)
             && Objects.equals(memoryUsage, other.memoryUsage)
+            && Objects.equals(analysisStats, other.analysisStats)
             && Objects.equals(node, other.node)
             && Objects.equals(assignmentExplanation, other.assignmentExplanation);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, state, failureReason, progress, memoryUsage, node, assignmentExplanation);
+        return Objects.hash(id, state, failureReason, progress, memoryUsage, analysisStats, node, assignmentExplanation);
     }
 
     @Override
@@ -152,6 +175,7 @@ public String toString() {
             .add("failureReason", failureReason)
             .add("progress", progress)
             .add("memoryUsage", memoryUsage)
+            .add("analysisStats", analysisStats)
             .add("node", node)
             .add("assignmentExplanation", assignmentExplanation)
             .toString();

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/AnalysisStats.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats;
+
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+/**
+ * Statistics for the data frame analysis
+ */
+public interface AnalysisStats extends ToXContentObject {
+
+    String getName();
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/AnalysisStatsNamedXContentProvider.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats;
+
+import org.elasticsearch.client.ml.dataframe.stats.classification.ClassificationStats;
+import org.elasticsearch.client.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
+import org.elasticsearch.client.ml.dataframe.stats.regression.RegressionStats;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class AnalysisStatsNamedXContentProvider implements NamedXContentProvider {
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        return Arrays.asList(
+            new NamedXContentRegistry.Entry(
+                AnalysisStats.class,
+                ClassificationStats.NAME,
+                (p, c) -> ClassificationStats.PARSER.apply(p, null)
+            ),
+            new NamedXContentRegistry.Entry(
+                AnalysisStats.class,
+                OutlierDetectionStats.NAME,
+                (p, c) -> OutlierDetectionStats.PARSER.apply(p, null)
+            ),
+            new NamedXContentRegistry.Entry(
+                AnalysisStats.class,
+                RegressionStats.NAME,
+                (p, c) -> RegressionStats.PARSER.apply(p, null)
+            )
+        );
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/stats/classification/ClassificationStats.java

@@ -0,0 +1,135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.stats.classification;
+
+import org.elasticsearch.client.common.TimeUtil;
+import org.elasticsearch.client.ml.dataframe.stats.AnalysisStats;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+public class ClassificationStats implements AnalysisStats {
+
+    public static final ParseField NAME = new ParseField("classification_stats");
+
+    public static final ParseField TIMESTAMP = new ParseField("timestamp");
+    public static final ParseField ITERATION = new ParseField("iteration");
+    public static final ParseField HYPERPARAMETERS = new ParseField("hyperparameters");
+    public static final ParseField TIMING_STATS = new ParseField("timing_stats");
+    public static final ParseField VALIDATION_LOSS = new ParseField("validation_loss");
+
+    public static final ConstructingObjectParser<ClassificationStats, Void> PARSER = new ConstructingObjectParser<>(NAME.getPreferredName(),
+        true,
+        a -> new ClassificationStats(
+            (Instant) a[0],
+            (Integer) a[1],
+            (Hyperparameters) a[2],
+            (TimingStats) a[3],
+            (ValidationLoss) a[4]
+        )
+    );
+
+    static {
+        PARSER.declareField(ConstructingObjectParser.constructorArg(),
+            p -> TimeUtil.parseTimeFieldToInstant(p, TIMESTAMP.getPreferredName()),
+            TIMESTAMP,
+            ObjectParser.ValueType.VALUE);
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), ITERATION);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), Hyperparameters.PARSER, HYPERPARAMETERS);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), TimingStats.PARSER, TIMING_STATS);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), ValidationLoss.PARSER, VALIDATION_LOSS);
+    }
+
+    private final Instant timestamp;
+    private final Integer iteration;
+    private final Hyperparameters hyperparameters;
+    private final TimingStats timingStats;
+    private final ValidationLoss validationLoss;
+
+    public ClassificationStats(Instant timestamp, Integer iteration, Hyperparameters hyperparameters, TimingStats timingStats,
+                               ValidationLoss validationLoss) {
+        this.timestamp = Instant.ofEpochMilli(Objects.requireNonNull(timestamp).toEpochMilli());
+        this.iteration = iteration;
+        this.hyperparameters = Objects.requireNonNull(hyperparameters);
+        this.timingStats = Objects.requireNonNull(timingStats);
+        this.validationLoss = Objects.requireNonNull(validationLoss);
+    }
+
+    public Instant getTimestamp() {
+        return timestamp;
+    }
+
+    public Integer getIteration() {
+        return iteration;
+    }
+
+    public Hyperparameters getHyperparameters() {
+        return hyperparameters;
+    }
+
+    public TimingStats getTimingStats() {
+        return timingStats;
+    }
+
+    public ValidationLoss getValidationLoss() {
+        return validationLoss;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
+        if (iteration != null) {
+            builder.field(ITERATION.getPreferredName(), iteration);
+        }
+        builder.field(HYPERPARAMETERS.getPreferredName(), hyperparameters);
+        builder.field(TIMING_STATS.getPreferredName(), timingStats);
+        builder.field(VALIDATION_LOSS.getPreferredName(), validationLoss);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ClassificationStats that = (ClassificationStats) o;
+        return Objects.equals(timestamp, that.timestamp)
+            && Objects.equals(iteration, that.iteration)
+            && Objects.equals(hyperparameters, that.hyperparameters)
+            && Objects.equals(timingStats, that.timingStats)
+            && Objects.equals(validationLoss, that.validationLoss);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(timestamp, iteration, hyperparameters, timingStats, validationLoss);
+    }
+
+    @Override
+    public String getName() {
+        return NAME.getPreferredName();
+    }
+}