Skip to content

Commit 8ff5131

Browse files
gibber9809gibber9809
committed
Add test for forwarded ArchiveStats
1 parent c425c98 commit 8ff5131

File tree

5 files changed

+118
-62
lines changed

5 files changed

+118
-62
lines changed

‎components/core/tests/clp_s_test_utils.cpp‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,21 @@
22

33
#include <filesystem>
44
#include <string>
5+
#include <vector>
56

67
#include <catch2/catch.hpp>
78

9+
#include "../src/clp_s/ArchiveWriter.hpp"
810
#include "../src/clp_s/InputConfig.hpp"
911
#include "../src/clp_s/JsonParser.hpp"
1012

11-
void compress_archive(
13+
auto compress_archive(
1214
std::string const& file_path,
1315
std::string const& archive_directory,
1416
bool single_file_archive,
1517
bool structurize_arrays,
1618
clp_s::FileType file_type
17-
) {
19+
) -> std::vector<clp_s::ArchiveStats> {
1820
constexpr auto cDefaultTargetEncodedSize{8ULL * 1024 * 1024 * 1024}; // 8 GiB
1921
constexpr auto cDefaultMaxDocumentSize{512ULL * 1024 * 1024}; // 512 MiB
2022
constexpr auto cDefaultMinTableSize{1ULL * 1024 * 1024}; // 1 MiB
@@ -39,6 +41,7 @@ void compress_archive(
3941
parser_option.input_file_type = file_type;
4042

4143
clp_s::JsonParser parser{parser_option};
44+
std::vector<clp_s::ArchiveStats> archive_stats;
4245
if (clp_s::FileType::Json == file_type) {
4346
REQUIRE(parser.parse());
4447
} else if (clp_s::FileType::KeyValueIr == file_type) {
@@ -47,7 +50,8 @@ void compress_archive(
4750
// This branch should be unreachable.
4851
REQUIRE(false);
4952
}
50-
REQUIRE_NOTHROW(parser.store());
53+
REQUIRE_NOTHROW(archive_stats = parser.store());
5154

5255
REQUIRE((false == std::filesystem::is_empty(archive_directory)));
56+
return archive_stats;
5357
}

‎components/core/tests/clp_s_test_utils.hpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
#ifndef CLP_S_TEST_UTILS_HPP
22
#define CLP_S_TEST_UTILS_HPP
3+
34
#include <string>
5+
#include <vector>
46

7+
#include "../src/clp_s/ArchiveWriter.hpp"
58
#include "../src/clp_s/InputConfig.hpp"
69

710
/**
@@ -14,12 +17,13 @@
1417
* @param single_file_archive
1518
* @param structurize_arrays
1619
* @param file_type
20+
* @return Statistics for every compressed archive.
1721
*/
18-
void compress_archive(
22+
[[nodiscard]] auto compress_archive(
1923
std::string const& file_path,
2024
std::string const& archive_directory,
2125
bool single_file_archive,
2226
bool structurize_arrays,
2327
clp_s::FileType file_type
24-
);
28+
) -> std::vector<clp_s::ArchiveStats>;
2529
#endif // CLP_S_TEST_UTILS_HPP

‎components/core/tests/test-clp_s-end_to_end.cpp‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <filesystem>
55
#include <string>
66
#include <string_view>
7+
#include <utility>
78

89
#include <catch2/catch.hpp>
910
#include <fmt/format.h>
@@ -102,13 +103,15 @@ TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") {
102103
std::string{cTestEndToEndOutputSortedJson}}
103104
};
104105

105-
REQUIRE_NOTHROW(compress_archive(
106-
get_test_input_local_path(),
107-
std::string{cTestEndToEndArchiveDirectory},
108-
single_file_archive,
109-
structurize_arrays,
110-
clp_s::FileType::Json
111-
));
106+
REQUIRE_NOTHROW(
107+
std::ignore = compress_archive(
108+
get_test_input_local_path(),
109+
std::string{cTestEndToEndArchiveDirectory},
110+
single_file_archive,
111+
structurize_arrays,
112+
clp_s::FileType::Json
113+
)
114+
);
112115

113116
auto extracted_json_path = extract();
114117

‎components/core/tests/test-clp_s-range_index.cpp‎

Lines changed: 86 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818
#include "../src/clp/type_utils.hpp"
1919
#include "../src/clp_s/archive_constants.hpp"
2020
#include "../src/clp_s/ArchiveReader.hpp"
21+
#include "../src/clp_s/ArchiveWriter.hpp"
2122
#include "../src/clp_s/InputConfig.hpp"
23+
#include "../src/clp_s/RangeIndexWriter.hpp"
2224
#include "clp_s_test_utils.hpp"
2325
#include "TestOutputCleaner.hpp"
2426

@@ -39,7 +41,13 @@ void serialize_record(
3941
clp::ffi::ir_stream::Serializer<clp::ir::eight_byte_encoded_variable_t>& serializer
4042
);
4143
void generate_ir();
42-
void check_archive_metadata(bool from_ir);
44+
void read_and_check_archive_metadata(bool from_ir);
45+
void check_archive_metadata_from_stats(
46+
std::vector<clp_s::ArchiveStats> const& archive_stats,
47+
bool from_ir
48+
);
49+
void
50+
check_archive_range_index(std::vector<clp_s::RangeIndexEntry> const& range_index, bool from_ir);
4351

4452
auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path {
4553
return std::filesystem::path{cTestRangeIndexInputFileDirectory} / cTestRangeIndexInputFile;
@@ -116,51 +124,82 @@ void generate_ir() {
116124
writer.close();
117125
}
118126

119-
void check_archive_metadata(bool from_ir) {
127+
void read_and_check_archive_metadata(bool from_ir) {
120128
clp_s::ArchiveReader archive_reader;
121-
auto const expected_input_path{
122-
from_ir ? get_ir_test_input_relative_path() : get_test_input_local_path()
123-
};
124129
for (auto const& entry : std::filesystem::directory_iterator(cTestRangeIndexArchiveDirectory)) {
125130
clp_s::Path archive_path{
126131
.source{clp_s::InputSource::Filesystem},
127132
.path{entry.path().string()}
128133
};
129134
REQUIRE_NOTHROW(archive_reader.open(archive_path, clp_s::NetworkAuthOption{}));
130135
auto const& range_index = archive_reader.get_range_index();
131-
REQUIRE(1ULL == range_index.size());
132-
auto const& range_index_entry = range_index.front();
133-
REQUIRE(0ULL == range_index_entry.start_index);
134-
REQUIRE(4ULL == range_index_entry.end_index);
135-
auto const& metadata_fields = range_index_entry.fields;
136-
REQUIRE(metadata_fields.contains(clp_s::constants::range_index::cArchiveCreatorId));
137-
REQUIRE(metadata_fields.at(clp_s::constants::range_index::cArchiveCreatorId).is_string());
138-
REQUIRE(false
139-
== metadata_fields.at(clp_s::constants::range_index::cArchiveCreatorId)
140-
.template get<std::string>()
141-
.empty());
142-
REQUIRE(metadata_fields.contains(clp_s::constants::range_index::cFilename));
143-
REQUIRE(metadata_fields.at(clp_s::constants::range_index::cFilename).is_string());
144-
REQUIRE(expected_input_path
145-
== metadata_fields.at(clp_s::constants::range_index::cFilename)
146-
.template get<std::string>());
147-
REQUIRE(metadata_fields.contains(clp_s::constants::range_index::cFileSplitNumber));
148-
REQUIRE(metadata_fields.at(clp_s::constants::range_index::cFileSplitNumber)
149-
.is_number_integer());
150-
REQUIRE(0ULL
151-
== metadata_fields.at(clp_s::constants::range_index::cFileSplitNumber)
152-
.template get<size_t>());
153-
if (from_ir) {
154-
REQUIRE(metadata_fields.contains(cTestRangeIndexIRMetadataKey));
155-
REQUIRE(metadata_fields.at(cTestRangeIndexIRMetadataKey).is_string());
156-
REQUIRE(
157-
cTestRangeIndexIRMetadataValue
158-
== metadata_fields.at(cTestRangeIndexIRMetadataKey).template get<std::string>()
159-
);
160-
}
136+
check_archive_range_index(range_index, from_ir);
161137
REQUIRE_NOTHROW(archive_reader.close());
162138
}
163139
}
140+
141+
void check_archive_metadata_from_stats(
142+
std::vector<clp_s::ArchiveStats> const& archive_stats,
143+
bool from_ir
144+
) {
145+
for (auto const& stats : archive_stats) {
146+
REQUIRE(stats.metadata.is_array());
147+
REQUIRE(1ULL == stats.metadata.size());
148+
auto entry = stats.metadata.begin();
149+
REQUIRE(entry->is_object());
150+
REQUIRE(entry->contains(clp_s::RangeIndexWriter::cStartIndexName));
151+
REQUIRE(entry->contains(clp_s::RangeIndexWriter::cEndIndexName));
152+
REQUIRE(entry->contains(clp_s::RangeIndexWriter::cMetadataFieldsName));
153+
auto start_index = entry->at(clp_s::RangeIndexWriter::cStartIndexName);
154+
auto end_index = entry->at(clp_s::RangeIndexWriter::cEndIndexName);
155+
auto metadata_fields = entry->at(clp_s::RangeIndexWriter::cMetadataFieldsName);
156+
REQUIRE(start_index.is_number_integer());
157+
REQUIRE(end_index.is_number_integer());
158+
REQUIRE(metadata_fields.is_object());
159+
std::vector<clp_s::RangeIndexEntry> range_index{clp_s::RangeIndexEntry{
160+
start_index.template get<size_t>(),
161+
end_index.template get<size_t>(),
162+
std::move(metadata_fields)
163+
}};
164+
check_archive_range_index(range_index, from_ir);
165+
}
166+
}
167+
168+
void
169+
check_archive_range_index(std::vector<clp_s::RangeIndexEntry> const& range_index, bool from_ir) {
170+
REQUIRE(1ULL == range_index.size());
171+
auto const& range_index_entry = range_index.front();
172+
REQUIRE(0ULL == range_index_entry.start_index);
173+
REQUIRE(4ULL == range_index_entry.end_index);
174+
auto const& metadata_fields = range_index_entry.fields;
175+
REQUIRE(metadata_fields.contains(clp_s::constants::range_index::cArchiveCreatorId));
176+
REQUIRE(metadata_fields.at(clp_s::constants::range_index::cArchiveCreatorId).is_string());
177+
REQUIRE(false
178+
== metadata_fields.at(clp_s::constants::range_index::cArchiveCreatorId)
179+
.template get<std::string>()
180+
.empty());
181+
REQUIRE(metadata_fields.contains(clp_s::constants::range_index::cFilename));
182+
REQUIRE(metadata_fields.at(clp_s::constants::range_index::cFilename).is_string());
183+
auto const expected_input_path{
184+
from_ir ? get_ir_test_input_relative_path() : get_test_input_local_path()
185+
};
186+
REQUIRE(expected_input_path
187+
== metadata_fields.at(clp_s::constants::range_index::cFilename)
188+
.template get<std::string>());
189+
REQUIRE(metadata_fields.contains(clp_s::constants::range_index::cFileSplitNumber));
190+
REQUIRE(
191+
metadata_fields.at(clp_s::constants::range_index::cFileSplitNumber).is_number_integer()
192+
);
193+
REQUIRE(0ULL
194+
== metadata_fields.at(clp_s::constants::range_index::cFileSplitNumber)
195+
.template get<size_t>());
196+
if (from_ir) {
197+
REQUIRE(metadata_fields.contains(cTestRangeIndexIRMetadataKey));
198+
REQUIRE(metadata_fields.at(cTestRangeIndexIRMetadataKey).is_string());
199+
REQUIRE(cTestRangeIndexIRMetadataValue
200+
== metadata_fields.at(cTestRangeIndexIRMetadataKey).template get<std::string>());
201+
}
202+
}
164203
} // namespace
165204

166205
TEST_CASE("clp-s-range-index", "[clp-s][range-index]") {
@@ -178,12 +217,16 @@ TEST_CASE("clp-s-range-index", "[clp-s][range-index]") {
178217
input_file = get_ir_test_input_relative_path();
179218
input_file_type = clp_s::FileType::KeyValueIr;
180219
}
181-
REQUIRE_NOTHROW(compress_archive(
182-
input_file,
183-
std::string{cTestRangeIndexArchiveDirectory},
184-
single_file_archive,
185-
false,
186-
input_file_type
187-
));
188-
check_archive_metadata(from_ir);
220+
std::vector<clp_s::ArchiveStats> archive_stats;
221+
REQUIRE_NOTHROW(
222+
archive_stats = compress_archive(
223+
input_file,
224+
std::string{cTestRangeIndexArchiveDirectory},
225+
single_file_archive,
226+
false,
227+
input_file_type
228+
)
229+
);
230+
read_and_check_archive_metadata(from_ir);
231+
check_archive_metadata_from_stats(archive_stats, from_ir);
189232
}

‎components/core/tests/test-clp_s-search.cpp‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -227,13 +227,15 @@ TEST_CASE("clp-s-search", "[clp-s][search]") {
227227

228228
TestOutputCleaner const test_cleanup{{std::string{cTestSearchArchiveDirectory}}};
229229

230-
REQUIRE_NOTHROW(compress_archive(
231-
get_test_input_local_path(),
232-
std::string{cTestSearchArchiveDirectory},
233-
single_file_archive,
234-
structurize_arrays,
235-
clp_s::FileType::Json
236-
));
230+
REQUIRE_NOTHROW(
231+
std::ignore = compress_archive(
232+
get_test_input_local_path(),
233+
std::string{cTestSearchArchiveDirectory},
234+
single_file_archive,
235+
structurize_arrays,
236+
clp_s::FileType::Json
237+
)
238+
);
237239

238240
for (auto const& [query, expected_results] : queries_and_results) {
239241
CAPTURE(query);

0 commit comments

Comments
 (0)