[ML] Add min_version to model snapshot

include/api/CModelSnapshotJsonWriter.h

@@ -41,6 +41,7 @@ class API_EXPORT CModelSnapshotJsonWriter
         //! Structure to store the model snapshot metadata
         struct SModelSnapshotReport
         {
+            std::string                       s_MinVersion;
             core_t::TTime                     s_SnapshotTimestamp;
             std::string                       s_Description;
             std::string                       s_SnapshotId;

lib/api/CAnomalyJob.cc

@@ -86,6 +86,10 @@ const std::string HIERARCHICAL_RESULTS_TAG("f");
 const std::string LATEST_RECORD_TIME_TAG("h");
 const std::string MODEL_PLOT_TAG("i");
 const std::string LAST_RESULTS_TIME_TAG("j");
+
+//! The minimum version required to read the state corresponding to a model snapshot.
+//! This should be updated every time there is a breaking change to the model state.
+const std::string MODEL_SNAPSHOT_MIN_VERSION("6.3.0");
 }
 
 // Statics
@@ -1372,6 +1376,7 @@ bool CAnomalyJob::persistState(const std::string &descriptionPrefix,
             if (m_PersistCompleteFunc)
             {
                 CModelSnapshotJsonWriter::SModelSnapshotReport modelSnapshotReport{
+                        MODEL_SNAPSHOT_MIN_VERSION,
                         snapshotTimestamp,
                         descriptionPrefix + core::CTimeUtils::toIso8601(snapshotTimestamp),
                         snapShotId,

lib/api/CModelSnapshotJsonWriter.cc

@@ -26,6 +26,7 @@ namespace
 
 // JSON field names
 const std::string JOB_ID("job_id");
+const std::string MIN_VERSION("min_version");
 const std::string TIMESTAMP("timestamp");
 const std::string MODEL_SNAPSHOT("model_snapshot");
 const std::string SNAPSHOT_ID("snapshot_id");
@@ -54,6 +55,8 @@ void CModelSnapshotJsonWriter::write(const SModelSnapshotReport &report)
 
     m_Writer.String(JOB_ID);
     m_Writer.String(m_JobId);
+    m_Writer.String(MIN_VERSION);
+    m_Writer.String(report.s_MinVersion);
     m_Writer.String(SNAPSHOT_ID);
     m_Writer.String(report.s_SnapshotId);
 

lib/api/unittest/CModelSnapshotJsonWriterTest.cc

@@ -0,0 +1,129 @@
+/*
+ * ELASTICSEARCH CONFIDENTIAL
+ *
+ * Copyright (c) 2018 Elasticsearch BV. All Rights Reserved.
+ *
+ * Notice: this software, and all information contained
+ * therein, is the exclusive property of Elasticsearch BV
+ * and its licensors, if any, and is protected under applicable
+ * domestic and foreign law, and international treaties.
+ *
+ * Reproduction, republication or distribution without the
+ * express written consent of Elasticsearch BV is
+ * strictly prohibited.
+ */
+#include "CModelSnapshotJsonWriterTest.h"
+
+#include <core/CJsonOutputStreamWrapper.h>
+#include <core/CoreTypes.h>
+
+#include <model/CResourceMonitor.h>
+
+#include <api/CModelSnapshotJsonWriter.h>
+
+#include <rapidjson/document.h>
+
+#include <string>
+
+using namespace ml;
+using namespace api;
+
+CppUnit::Test *CModelSnapshotJsonWriterTest::suite()
+{
+    CppUnit::TestSuite *suiteOfTests = new CppUnit::TestSuite("CModelSnapshotJsonWriterTest");
+    suiteOfTests->addTest( new CppUnit::TestCaller<CModelSnapshotJsonWriterTest>(
+                                   "CModelSnapshotJsonWriterTest::testWrite",
+                                   &CModelSnapshotJsonWriterTest::testWrite) );
+    return suiteOfTests;
+}
+
+void CModelSnapshotJsonWriterTest::testWrite(void)
+{
+    std::ostringstream sstream;
+
+    // The output writer won't close the JSON structures until is is destroyed
+    {
+        model::CResourceMonitor::SResults modelSizeStats{
+            10000, // bytes used
+            3, // # by fields
+            1, // # partition fields
+            150, // # over fields
+            4, // # allocation failures
+            model_t::E_MemoryStatusOk, // memory status
+            core_t::TTime(1521046309) // bucket start time
+        };
+
+        CModelSnapshotJsonWriter::SModelSnapshotReport report{
+            "6.3.0",
+            core_t::TTime(1521046309),
+            "the snapshot description",
+            "test_snapshot_id",
+            size_t(15), // # docs
+            modelSizeStats,
+            "some normalizer state",
+            core_t::TTime(1521046409), // last record time
+            core_t::TTime(1521040000) // last result time
+        };
+
+        core::CJsonOutputStreamWrapper wrappedOutStream(sstream);
+        CModelSnapshotJsonWriter writer("job", wrappedOutStream);
+        writer.write(report);
+    }
+
+    rapidjson::Document arrayDoc;
+    arrayDoc.Parse<rapidjson::kParseDefaultFlags>(sstream.str().c_str());
+
+    CPPUNIT_ASSERT(arrayDoc.IsArray());
+    CPPUNIT_ASSERT_EQUAL(rapidjson::SizeType(1), arrayDoc.Size());
+
+    const rapidjson::Value &object = arrayDoc[rapidjson::SizeType(0)];
+    CPPUNIT_ASSERT(object.IsObject());
+
+    CPPUNIT_ASSERT(object.HasMember("model_snapshot"));
+    const rapidjson::Value &snapshot = object["model_snapshot"];
+    CPPUNIT_ASSERT(snapshot.HasMember("job_id"));
+    CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(snapshot["job_id"].GetString()));
+    CPPUNIT_ASSERT(snapshot.HasMember("min_version"));
+    CPPUNIT_ASSERT_EQUAL(std::string("6.3.0"), std::string(snapshot["min_version"].GetString()));
+    CPPUNIT_ASSERT(snapshot.HasMember("snapshot_id"));
+    CPPUNIT_ASSERT_EQUAL(std::string("test_snapshot_id"), std::string(snapshot["snapshot_id"].GetString()));
+    CPPUNIT_ASSERT(snapshot.HasMember("snapshot_doc_count"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(15), snapshot["snapshot_doc_count"].GetInt64());
+    CPPUNIT_ASSERT(snapshot.HasMember("timestamp"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(1521046309000), snapshot["timestamp"].GetInt64());
+    CPPUNIT_ASSERT(snapshot.HasMember("description"));
+    CPPUNIT_ASSERT_EQUAL(std::string("the snapshot description"), std::string(snapshot["description"].GetString()));
+    CPPUNIT_ASSERT(snapshot.HasMember("latest_record_time_stamp"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(1521046409000), snapshot["latest_record_time_stamp"].GetInt64());
+    CPPUNIT_ASSERT(snapshot.HasMember("latest_result_time_stamp"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(1521040000000), snapshot["latest_result_time_stamp"].GetInt64());
+
+    CPPUNIT_ASSERT(snapshot.HasMember("model_size_stats"));
+    const rapidjson::Value &modelSizeStats = snapshot["model_size_stats"];
+    CPPUNIT_ASSERT(modelSizeStats.HasMember("job_id"));
+    CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(modelSizeStats["job_id"].GetString()));
+    CPPUNIT_ASSERT(modelSizeStats.HasMember("model_bytes"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(20000), modelSizeStats["model_bytes"].GetInt64());
+    CPPUNIT_ASSERT(modelSizeStats.HasMember("total_by_field_count"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(3), modelSizeStats["total_by_field_count"].GetInt64());
+    CPPUNIT_ASSERT(modelSizeStats.HasMember("total_partition_field_count"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(1), modelSizeStats["total_partition_field_count"].GetInt64());
+    CPPUNIT_ASSERT(modelSizeStats.HasMember("total_over_field_count"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(150), modelSizeStats["total_over_field_count"].GetInt64());
+    CPPUNIT_ASSERT(modelSizeStats.HasMember("bucket_allocation_failures_count"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(4), modelSizeStats["bucket_allocation_failures_count"].GetInt64());
+    CPPUNIT_ASSERT(modelSizeStats.HasMember("memory_status"));
+    CPPUNIT_ASSERT_EQUAL(std::string("ok"), std::string(modelSizeStats["memory_status"].GetString()));
+    CPPUNIT_ASSERT(modelSizeStats.HasMember("timestamp"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(1521046309000), modelSizeStats["timestamp"].GetInt64());
+    CPPUNIT_ASSERT(modelSizeStats.HasMember("log_time"));
+
+    CPPUNIT_ASSERT(snapshot.HasMember("quantiles"));
+    const rapidjson::Value &quantiles = snapshot["quantiles"];
+    CPPUNIT_ASSERT(quantiles.HasMember("job_id"));
+    CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(quantiles["job_id"].GetString()));
+    CPPUNIT_ASSERT(quantiles.HasMember("quantile_state"));
+    CPPUNIT_ASSERT_EQUAL(std::string("some normalizer state"), std::string(quantiles["quantile_state"].GetString()));
+    CPPUNIT_ASSERT(quantiles.HasMember("timestamp"));
+    CPPUNIT_ASSERT_EQUAL(int64_t(1521040000000), quantiles["timestamp"].GetInt64());
+}

lib/api/unittest/CModelSnapshotJsonWriterTest.h

@@ -0,0 +1,30 @@
+/*
+ * ELASTICSEARCH CONFIDENTIAL
+ *
+ * Copyright (c) 2018 Elasticsearch BV. All Rights Reserved.
+ *
+ * Notice: this software, and all information contained
+ * therein, is the exclusive property of Elasticsearch BV
+ * and its licensors, if any, and is protected under applicable
+ * domestic and foreign law, and international treaties.
+ *
+ * Reproduction, republication or distribution without the
+ * express written consent of Elasticsearch BV is
+ * strictly prohibited.
+ */
+#ifndef INCLUDED_CModelSnapshotJsonWriterTest_h
+#define INCLUDED_CModelSnapshotJsonWriterTest_h
+
+#include <cppunit/extensions/HelperMacros.h>
+
+
+class CModelSnapshotJsonWriterTest : public CppUnit::TestFixture
+{
+    public:
+        void testWrite(void);
+
+        static CppUnit::Test *suite();
+};
+
+#endif // INCLUDED_CModelSnapshotJsonWriterTest_h
+

lib/api/unittest/Main.cc

@@ -33,6 +33,7 @@
 #include "CLineifiedJsonOutputWriterTest.h"
 #include "CLineifiedXmlInputParserTest.h"
 #include "CModelPlotDataJsonWriterTest.h"
+#include "CModelSnapshotJsonWriterTest.h"
 #include "CMultiFileDataAdderTest.h"
 #include "COutputChainerTest.h"
 #include "CRestorePreviousStateTest.h"
@@ -67,6 +68,7 @@ int main(int argc, const char **argv)
     runner.addTest( CLineifiedJsonOutputWriterTest::suite() );
     runner.addTest( CLineifiedXmlInputParserTest::suite() );
     runner.addTest( CModelPlotDataJsonWriterTest::suite() );
+    runner.addTest( CModelSnapshotJsonWriterTest::suite() );
     runner.addTest( CMultiFileDataAdderTest::suite() );
     runner.addTest( COutputChainerTest::suite() );
     runner.addTest( CRestorePreviousStateTest::suite() );

lib/api/unittest/Makefile

@@ -51,6 +51,7 @@ SRCS=\
 	CMockDataProcessor.cc \
 	CMockSearcher.cc \
 	CModelPlotDataJsonWriterTest.cc \
+	CModelSnapshotJsonWriterTest.cc \
 	CMultiFileDataAdderTest.cc \
 	COutputChainerTest.cc \
 	CRestorePreviousStateTest.cc \