Skip to content

Commit fa40f99

Browse files
dimitris-athanasioudimitris-athanasiou
committed
[ML] Add min_version to model snapshot (#17)
1 parent 1e09684 commit fa40f99

File tree

7 files changed

+153
-0
lines changed

7 files changed

+153
-0
lines changed

‎include/api/CModelSnapshotJsonWriter.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ class API_EXPORT CModelSnapshotJsonWriter
3232
//! Structure to store the model snapshot metadata
3333
struct SModelSnapshotReport
3434
{
35+
std::string s_MinVersion;
3536
core_t::TTime s_SnapshotTimestamp;
3637
std::string s_Description;
3738
std::string s_SnapshotId;

‎lib/api/CAnomalyJob.cc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,10 @@ const std::string HIERARCHICAL_RESULTS_TAG("f");
7777
const std::string LATEST_RECORD_TIME_TAG("h");
7878
const std::string MODEL_PLOT_TAG("i");
7979
const std::string LAST_RESULTS_TIME_TAG("j");
80+
81+
//! The minimum version required to read the state corresponding to a model snapshot.
82+
//! This should be updated every time there is a breaking change to the model state.
83+
const std::string MODEL_SNAPSHOT_MIN_VERSION("6.3.0");
8084
}
8185

8286
// Statics
@@ -1363,6 +1367,7 @@ bool CAnomalyJob::persistState(const std::string &descriptionPrefix,
13631367
if (m_PersistCompleteFunc)
13641368
{
13651369
CModelSnapshotJsonWriter::SModelSnapshotReport modelSnapshotReport{
1370+
MODEL_SNAPSHOT_MIN_VERSION,
13661371
snapshotTimestamp,
13671372
descriptionPrefix + core::CTimeUtils::toIso8601(snapshotTimestamp),
13681373
snapShotId,

‎lib/api/CModelSnapshotJsonWriter.cc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ namespace
1717

1818
// JSON field names
1919
const std::string JOB_ID("job_id");
20+
const std::string MIN_VERSION("min_version");
2021
const std::string TIMESTAMP("timestamp");
2122
const std::string MODEL_SNAPSHOT("model_snapshot");
2223
const std::string SNAPSHOT_ID("snapshot_id");
@@ -45,6 +46,8 @@ void CModelSnapshotJsonWriter::write(const SModelSnapshotReport &report)
4546

4647
m_Writer.String(JOB_ID);
4748
m_Writer.String(m_JobId);
49+
m_Writer.String(MIN_VERSION);
50+
m_Writer.String(report.s_MinVersion);
4851
m_Writer.String(SNAPSHOT_ID);
4952
m_Writer.String(report.s_SnapshotId);
5053

‎lib/api/unittest/CModelSnapshotJsonWriterTest.cc

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
#include "CModelSnapshotJsonWriterTest.h"
7+
8+
#include <core/CJsonOutputStreamWrapper.h>
9+
#include <core/CoreTypes.h>
10+
11+
#include <model/CResourceMonitor.h>
12+
13+
#include <api/CModelSnapshotJsonWriter.h>
14+
15+
#include <rapidjson/document.h>
16+
17+
#include <string>
18+
19+
using namespace ml;
20+
using namespace api;
21+
22+
CppUnit::Test *CModelSnapshotJsonWriterTest::suite()
23+
{
24+
CppUnit::TestSuite *suiteOfTests = new CppUnit::TestSuite("CModelSnapshotJsonWriterTest");
25+
suiteOfTests->addTest( new CppUnit::TestCaller<CModelSnapshotJsonWriterTest>(
26+
"CModelSnapshotJsonWriterTest::testWrite",
27+
&CModelSnapshotJsonWriterTest::testWrite) );
28+
return suiteOfTests;
29+
}
30+
31+
void CModelSnapshotJsonWriterTest::testWrite(void)
32+
{
33+
std::ostringstream sstream;
34+
35+
// The output writer won't close the JSON structures until is is destroyed
36+
{
37+
model::CResourceMonitor::SResults modelSizeStats{
38+
10000, // bytes used
39+
3, // # by fields
40+
1, // # partition fields
41+
150, // # over fields
42+
4, // # allocation failures
43+
model_t::E_MemoryStatusOk, // memory status
44+
core_t::TTime(1521046309) // bucket start time
45+
};
46+
47+
CModelSnapshotJsonWriter::SModelSnapshotReport report{
48+
"6.3.0",
49+
core_t::TTime(1521046309),
50+
"the snapshot description",
51+
"test_snapshot_id",
52+
size_t(15), // # docs
53+
modelSizeStats,
54+
"some normalizer state",
55+
core_t::TTime(1521046409), // last record time
56+
core_t::TTime(1521040000) // last result time
57+
};
58+
59+
core::CJsonOutputStreamWrapper wrappedOutStream(sstream);
60+
CModelSnapshotJsonWriter writer("job", wrappedOutStream);
61+
writer.write(report);
62+
}
63+
64+
rapidjson::Document arrayDoc;
65+
arrayDoc.Parse<rapidjson::kParseDefaultFlags>(sstream.str().c_str());
66+
67+
CPPUNIT_ASSERT(arrayDoc.IsArray());
68+
CPPUNIT_ASSERT_EQUAL(rapidjson::SizeType(1), arrayDoc.Size());
69+
70+
const rapidjson::Value &object = arrayDoc[rapidjson::SizeType(0)];
71+
CPPUNIT_ASSERT(object.IsObject());
72+
73+
CPPUNIT_ASSERT(object.HasMember("model_snapshot"));
74+
const rapidjson::Value &snapshot = object["model_snapshot"];
75+
CPPUNIT_ASSERT(snapshot.HasMember("job_id"));
76+
CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(snapshot["job_id"].GetString()));
77+
CPPUNIT_ASSERT(snapshot.HasMember("min_version"));
78+
CPPUNIT_ASSERT_EQUAL(std::string("6.3.0"), std::string(snapshot["min_version"].GetString()));
79+
CPPUNIT_ASSERT(snapshot.HasMember("snapshot_id"));
80+
CPPUNIT_ASSERT_EQUAL(std::string("test_snapshot_id"), std::string(snapshot["snapshot_id"].GetString()));
81+
CPPUNIT_ASSERT(snapshot.HasMember("snapshot_doc_count"));
82+
CPPUNIT_ASSERT_EQUAL(int64_t(15), snapshot["snapshot_doc_count"].GetInt64());
83+
CPPUNIT_ASSERT(snapshot.HasMember("timestamp"));
84+
CPPUNIT_ASSERT_EQUAL(int64_t(1521046309000), snapshot["timestamp"].GetInt64());
85+
CPPUNIT_ASSERT(snapshot.HasMember("description"));
86+
CPPUNIT_ASSERT_EQUAL(std::string("the snapshot description"), std::string(snapshot["description"].GetString()));
87+
CPPUNIT_ASSERT(snapshot.HasMember("latest_record_time_stamp"));
88+
CPPUNIT_ASSERT_EQUAL(int64_t(1521046409000), snapshot["latest_record_time_stamp"].GetInt64());
89+
CPPUNIT_ASSERT(snapshot.HasMember("latest_result_time_stamp"));
90+
CPPUNIT_ASSERT_EQUAL(int64_t(1521040000000), snapshot["latest_result_time_stamp"].GetInt64());
91+
92+
CPPUNIT_ASSERT(snapshot.HasMember("model_size_stats"));
93+
const rapidjson::Value &modelSizeStats = snapshot["model_size_stats"];
94+
CPPUNIT_ASSERT(modelSizeStats.HasMember("job_id"));
95+
CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(modelSizeStats["job_id"].GetString()));
96+
CPPUNIT_ASSERT(modelSizeStats.HasMember("model_bytes"));
97+
CPPUNIT_ASSERT_EQUAL(int64_t(20000), modelSizeStats["model_bytes"].GetInt64());
98+
CPPUNIT_ASSERT(modelSizeStats.HasMember("total_by_field_count"));
99+
CPPUNIT_ASSERT_EQUAL(int64_t(3), modelSizeStats["total_by_field_count"].GetInt64());
100+
CPPUNIT_ASSERT(modelSizeStats.HasMember("total_partition_field_count"));
101+
CPPUNIT_ASSERT_EQUAL(int64_t(1), modelSizeStats["total_partition_field_count"].GetInt64());
102+
CPPUNIT_ASSERT(modelSizeStats.HasMember("total_over_field_count"));
103+
CPPUNIT_ASSERT_EQUAL(int64_t(150), modelSizeStats["total_over_field_count"].GetInt64());
104+
CPPUNIT_ASSERT(modelSizeStats.HasMember("bucket_allocation_failures_count"));
105+
CPPUNIT_ASSERT_EQUAL(int64_t(4), modelSizeStats["bucket_allocation_failures_count"].GetInt64());
106+
CPPUNIT_ASSERT(modelSizeStats.HasMember("memory_status"));
107+
CPPUNIT_ASSERT_EQUAL(std::string("ok"), std::string(modelSizeStats["memory_status"].GetString()));
108+
CPPUNIT_ASSERT(modelSizeStats.HasMember("timestamp"));
109+
CPPUNIT_ASSERT_EQUAL(int64_t(1521046309000), modelSizeStats["timestamp"].GetInt64());
110+
CPPUNIT_ASSERT(modelSizeStats.HasMember("log_time"));
111+
112+
CPPUNIT_ASSERT(snapshot.HasMember("quantiles"));
113+
const rapidjson::Value &quantiles = snapshot["quantiles"];
114+
CPPUNIT_ASSERT(quantiles.HasMember("job_id"));
115+
CPPUNIT_ASSERT_EQUAL(std::string("job"), std::string(quantiles["job_id"].GetString()));
116+
CPPUNIT_ASSERT(quantiles.HasMember("quantile_state"));
117+
CPPUNIT_ASSERT_EQUAL(std::string("some normalizer state"), std::string(quantiles["quantile_state"].GetString()));
118+
CPPUNIT_ASSERT(quantiles.HasMember("timestamp"));
119+
CPPUNIT_ASSERT_EQUAL(int64_t(1521040000000), quantiles["timestamp"].GetInt64());
120+
}

‎lib/api/unittest/CModelSnapshotJsonWriterTest.h

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
#ifndef INCLUDED_CModelSnapshotJsonWriterTest_h
7+
#define INCLUDED_CModelSnapshotJsonWriterTest_h
8+
9+
#include <cppunit/extensions/HelperMacros.h>
10+
11+
12+
class CModelSnapshotJsonWriterTest : public CppUnit::TestFixture
13+
{
14+
public:
15+
void testWrite(void);
16+
17+
static CppUnit::Test *suite();
18+
};
19+
20+
#endif // INCLUDED_CModelSnapshotJsonWriterTest_h
21+

‎lib/api/unittest/Main.cc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "CLineifiedJsonOutputWriterTest.h"
2525
#include "CLineifiedXmlInputParserTest.h"
2626
#include "CModelPlotDataJsonWriterTest.h"
27+
#include "CModelSnapshotJsonWriterTest.h"
2728
#include "CMultiFileDataAdderTest.h"
2829
#include "COutputChainerTest.h"
2930
#include "CRestorePreviousStateTest.h"
@@ -58,6 +59,7 @@ int main(int argc, const char **argv)
5859
runner.addTest( CLineifiedJsonOutputWriterTest::suite() );
5960
runner.addTest( CLineifiedXmlInputParserTest::suite() );
6061
runner.addTest( CModelPlotDataJsonWriterTest::suite() );
62+
runner.addTest( CModelSnapshotJsonWriterTest::suite() );
6163
runner.addTest( CMultiFileDataAdderTest::suite() );
6264
runner.addTest( COutputChainerTest::suite() );
6365
runner.addTest( CRestorePreviousStateTest::suite() );

‎lib/api/unittest/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ SRCS=\
4242
CMockDataProcessor.cc \
4343
CMockSearcher.cc \
4444
CModelPlotDataJsonWriterTest.cc \
45+
CModelSnapshotJsonWriterTest.cc \
4546
CMultiFileDataAdderTest.cc \
4647
COutputChainerTest.cc \
4748
CRestorePreviousStateTest.cc \

0 commit comments

Comments
 (0)