Skip to content

Commit 047a4f8

Browse files
dimitris-athanasioudimitris-athanasiou
authored
[ML] Add min_version to model snapshot (#17)
1 parent 9cc58e6 commit 047a4f8

File tree

7 files changed

+171
-0
lines changed

7 files changed

+171
-0
lines changed

‎include/api/CModelSnapshotJsonWriter.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ class API_EXPORT CModelSnapshotJsonWriter
4141
//! Structure to store the model snapshot metadata
4242
struct SModelSnapshotReport
4343
{
44+
std::string s_MinVersion;
4445
core_t::TTime s_SnapshotTimestamp;
4546
std::string s_Description;
4647
std::string s_SnapshotId;

‎lib/api/CAnomalyJob.cc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ const std::string HIERARCHICAL_RESULTS_TAG("f");
8686
const std::string LATEST_RECORD_TIME_TAG("h");
8787
const std::string MODEL_PLOT_TAG("i");
8888
const std::string LAST_RESULTS_TIME_TAG("j");
89+
90+
//! The minimum version required to read the state corresponding to a model snapshot.
91+
//! This should be updated every time there is a breaking change to the model state.
92+
const std::string MODEL_SNAPSHOT_MIN_VERSION("6.3.0");
8993
}
9094

9195
// Statics
@@ -1372,6 +1376,7 @@ bool CAnomalyJob::persistState(const std::string &descriptionPrefix,
13721376
if (m_PersistCompleteFunc)
13731377
{
13741378
CModelSnapshotJsonWriter::SModelSnapshotReport modelSnapshotReport{
1379+
MODEL_SNAPSHOT_MIN_VERSION,
13751380
snapshotTimestamp,
13761381
descriptionPrefix + core::CTimeUtils::toIso8601(snapshotTimestamp),
13771382
snapShotId,

‎lib/api/CModelSnapshotJsonWriter.cc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ namespace
2626

2727
// JSON field names
2828
const std::string JOB_ID("job_id");
29+
const std::string MIN_VERSION("min_version");
2930
const std::string TIMESTAMP("timestamp");
3031
const std::string MODEL_SNAPSHOT("model_snapshot");
3132
const std::string SNAPSHOT_ID("snapshot_id");
@@ -54,6 +55,8 @@ void CModelSnapshotJsonWriter::write(const SModelSnapshotReport &report)
5455

5556
m_Writer.String(JOB_ID);
5657
m_Writer.String(m_JobId);
58+
m_Writer.String(MIN_VERSION);
59+
m_Writer.String(report.s_MinVersion);
5760
m_Writer.String(SNAPSHOT_ID);
5861
m_Writer.String(report.s_SnapshotId);
5962

‎lib/api/unittest/CModelSnapshotJsonWriterTest.cc

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/*
2+
* ELASTICSEARCH CONFIDENTIAL
3+
*
4+
* Copyright (c) 2018 Elasticsearch BV. All Rights Reserved.
5+
*
6+
* Notice: this software, and all information contained
7+
* therein, is the exclusive property of Elasticsearch BV
8+
* and its licensors, if any, and is protected under applicable
9+
* domestic and foreign law, and international treaties.
10+
*
11+
* Reproduction, republication or distribution without the
12+
* express written consent of Elasticsearch BV is
13+
* strictly prohibited.
14+
*/
15+
#include "CModelSnapshotJsonWriterTest.h"
16+
17+
#include <core/CJsonOutputStreamWrapper.h>
18+
#include <core/CoreTypes.h>
19+
20+
#include <model/CResourceMonitor.h>
21+
22+
#include <api/CModelSnapshotJsonWriter.h>
23+
24+
#include <rapidjson/document.h>
25+
26+
#include <string>
27+
28+
using namespace ml;
29+
using namespace api;
30+
31+
CppUnit::Test *CModelSnapshotJsonWriterTest::suite()
32+
{
33+
CppUnit::TestSuite *suiteOfTests = new CppUnit::TestSuite("CModelSnapshotJsonWriterTest");
34+
suiteOfTests->addTest( new CppUnit::TestCaller<CModelSnapshotJsonWriterTest>(
35+
"CModelSnapshotJsonWriterTest::testWrite",
36+
&CModelSnapshotJsonWriterTest::testWrite) );
37+
return suiteOfTests;
38+
}
39+
40+
void CModelSnapshotJsonWriterTest::testWrite(void)
41+
{
42+
std::ostringstream sstream;
43+
44+
// The output writer won't close the JSON structures until is is destroyed
45+
{
46+
model::CResourceMonitor::SResults modelSizeStats{
47+
10000, // bytes used
48+
3, // # by fields
49+
1, // # partition fields
50+
150, // # over fields
51+
4, // # allocation failures
52+
model_t::E_MemoryStatusOk, // memory status
53+
core_t::TTime(1521046309) // bucket start time
54+
};
55+
56+
CModelSnapshotJsonWriter::SModelSnapshotReport report{
57+
"6.3.0",
58+
core_t::TTime(1521046309),
59+
"the snapshot description",
60+
"test_snapshot_id",
61+
size_t(15), // # docs
62+
modelSizeStats,
63+
"some normalizer state",
64+
core_t::TTime(1521046409), // last record time
65+
core_t::TTime(1521040000) // last result time
66+
};
67+
68+
core::CJsonOutputStreamWrapper wrappedOutStream(sstream);
69+
CModelSnapshotJsonWriter writer("job", wrappedOutStream);
70+
writer.write(report);
71+
}
72+
73+
rapidjson::Document arrayDoc;
74+
arrayDoc.Parse<rapidjson::kParseDefaultFlags>(sstream.str().c_str());
75+
76+
CPPUNIT_ASSERT(arrayDoc.IsArray());
77+
CPPUNIT_ASSERT_EQUAL(rapidjson::SizeType(1), arrayDoc.Size());
78+
79+
const rapidjson::Value &object = arrayDoc[rapidjson::SizeType(0)];
80+
CPPUNIT_ASSERT(object.IsObject());
81+
82+
CPPUNIT_ASSERT(object.HasMember("model_snapshot"));
83+
const rapidjson::Value &snapshot = object["model_snapshot"];
84+
CPPUNIT_ASSERT(snapshot.HasMember("job_id"));
85+
CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(snapshot["job_id"].GetString()));
86+
CPPUNIT_ASSERT(snapshot.HasMember("min_version"));
87+
CPPUNIT_ASSERT_EQUAL(std::string("6.3.0"), std::string(snapshot["min_version"].GetString()));
88+
CPPUNIT_ASSERT(snapshot.HasMember("snapshot_id"));
89+
CPPUNIT_ASSERT_EQUAL(std::string("test_snapshot_id"), std::string(snapshot["snapshot_id"].GetString()));
90+
CPPUNIT_ASSERT(snapshot.HasMember("snapshot_doc_count"));
91+
CPPUNIT_ASSERT_EQUAL(int64_t(15), snapshot["snapshot_doc_count"].GetInt64());
92+
CPPUNIT_ASSERT(snapshot.HasMember("timestamp"));
93+
CPPUNIT_ASSERT_EQUAL(int64_t(1521046309000), snapshot["timestamp"].GetInt64());
94+
CPPUNIT_ASSERT(snapshot.HasMember("description"));
95+
CPPUNIT_ASSERT_EQUAL(std::string("the snapshot description"), std::string(snapshot["description"].GetString()));
96+
CPPUNIT_ASSERT(snapshot.HasMember("latest_record_time_stamp"));
97+
CPPUNIT_ASSERT_EQUAL(int64_t(1521046409000), snapshot["latest_record_time_stamp"].GetInt64());
98+
CPPUNIT_ASSERT(snapshot.HasMember("latest_result_time_stamp"));
99+
CPPUNIT_ASSERT_EQUAL(int64_t(1521040000000), snapshot["latest_result_time_stamp"].GetInt64());
100+
101+
CPPUNIT_ASSERT(snapshot.HasMember("model_size_stats"));
102+
const rapidjson::Value &modelSizeStats = snapshot["model_size_stats"];
103+
CPPUNIT_ASSERT(modelSizeStats.HasMember("job_id"));
104+
CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(modelSizeStats["job_id"].GetString()));
105+
CPPUNIT_ASSERT(modelSizeStats.HasMember("model_bytes"));
106+
CPPUNIT_ASSERT_EQUAL(int64_t(20000), modelSizeStats["model_bytes"].GetInt64());
107+
CPPUNIT_ASSERT(modelSizeStats.HasMember("total_by_field_count"));
108+
CPPUNIT_ASSERT_EQUAL(int64_t(3), modelSizeStats["total_by_field_count"].GetInt64());
109+
CPPUNIT_ASSERT(modelSizeStats.HasMember("total_partition_field_count"));
110+
CPPUNIT_ASSERT_EQUAL(int64_t(1), modelSizeStats["total_partition_field_count"].GetInt64());
111+
CPPUNIT_ASSERT(modelSizeStats.HasMember("total_over_field_count"));
112+
CPPUNIT_ASSERT_EQUAL(int64_t(150), modelSizeStats["total_over_field_count"].GetInt64());
113+
CPPUNIT_ASSERT(modelSizeStats.HasMember("bucket_allocation_failures_count"));
114+
CPPUNIT_ASSERT_EQUAL(int64_t(4), modelSizeStats["bucket_allocation_failures_count"].GetInt64());
115+
CPPUNIT_ASSERT(modelSizeStats.HasMember("memory_status"));
116+
CPPUNIT_ASSERT_EQUAL(std::string("ok"), std::string(modelSizeStats["memory_status"].GetString()));
117+
CPPUNIT_ASSERT(modelSizeStats.HasMember("timestamp"));
118+
CPPUNIT_ASSERT_EQUAL(int64_t(1521046309000), modelSizeStats["timestamp"].GetInt64());
119+
CPPUNIT_ASSERT(modelSizeStats.HasMember("log_time"));
120+
121+
CPPUNIT_ASSERT(snapshot.HasMember("quantiles"));
122+
const rapidjson::Value &quantiles = snapshot["quantiles"];
123+
CPPUNIT_ASSERT(quantiles.HasMember("job_id"));
124+
CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(quantiles["job_id"].GetString()));
125+
CPPUNIT_ASSERT(quantiles.HasMember("quantile_state"));
126+
CPPUNIT_ASSERT_EQUAL(std::string("some normalizer state"), std::string(quantiles["quantile_state"].GetString()));
127+
CPPUNIT_ASSERT(quantiles.HasMember("timestamp"));
128+
CPPUNIT_ASSERT_EQUAL(int64_t(1521040000000), quantiles["timestamp"].GetInt64());
129+
}

‎lib/api/unittest/CModelSnapshotJsonWriterTest.h

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* ELASTICSEARCH CONFIDENTIAL
3+
*
4+
* Copyright (c) 2018 Elasticsearch BV. All Rights Reserved.
5+
*
6+
* Notice: this software, and all information contained
7+
* therein, is the exclusive property of Elasticsearch BV
8+
* and its licensors, if any, and is protected under applicable
9+
* domestic and foreign law, and international treaties.
10+
*
11+
* Reproduction, republication or distribution without the
12+
* express written consent of Elasticsearch BV is
13+
* strictly prohibited.
14+
*/
15+
#ifndef INCLUDED_CModelSnapshotJsonWriterTest_h
16+
#define INCLUDED_CModelSnapshotJsonWriterTest_h
17+
18+
#include <cppunit/extensions/HelperMacros.h>
19+
20+
21+
class CModelSnapshotJsonWriterTest : public CppUnit::TestFixture
22+
{
23+
public:
24+
void testWrite(void);
25+
26+
static CppUnit::Test *suite();
27+
};
28+
29+
#endif // INCLUDED_CModelSnapshotJsonWriterTest_h
30+

‎lib/api/unittest/Main.cc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "CLineifiedJsonOutputWriterTest.h"
3434
#include "CLineifiedXmlInputParserTest.h"
3535
#include "CModelPlotDataJsonWriterTest.h"
36+
#include "CModelSnapshotJsonWriterTest.h"
3637
#include "CMultiFileDataAdderTest.h"
3738
#include "COutputChainerTest.h"
3839
#include "CRestorePreviousStateTest.h"
@@ -67,6 +68,7 @@ int main(int argc, const char **argv)
6768
runner.addTest( CLineifiedJsonOutputWriterTest::suite() );
6869
runner.addTest( CLineifiedXmlInputParserTest::suite() );
6970
runner.addTest( CModelPlotDataJsonWriterTest::suite() );
71+
runner.addTest( CModelSnapshotJsonWriterTest::suite() );
7072
runner.addTest( CMultiFileDataAdderTest::suite() );
7173
runner.addTest( COutputChainerTest::suite() );
7274
runner.addTest( CRestorePreviousStateTest::suite() );

‎lib/api/unittest/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ SRCS=\
5151
CMockDataProcessor.cc \
5252
CMockSearcher.cc \
5353
CModelPlotDataJsonWriterTest.cc \
54+
CModelSnapshotJsonWriterTest.cc \
5455
CMultiFileDataAdderTest.cc \
5556
COutputChainerTest.cc \
5657
CRestorePreviousStateTest.cc \

0 commit comments

Comments
 (0)