Skip to content

fix: missing 00000 immutable files with Cardano DB v2 #2520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
May 26, 2025
Merged

fix: missing 00000 immutable files with Cardano DB v2 #2520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
434decc
fix(aggregator): add missing '00000' immutable files
jpraynaud May 26, 2025
d25b8b2
fix(aggregator): failing test of Cardano DB v2 artifact builder
jpraynaud May 26, 2025
134fd58
fix(client): retrieve '00000' immutable files in client
jpraynaud May 26, 2025
5bf98fe
fix(ci): Mithril Client multi-platform test Cardano DB restoration fa…
jpraynaud May 26, 2025
9a5e305
chore: upgrade crate versions
jpraynaud May 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test-client.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ jobs:
if: steps.aggregator_capability_unix.outputs.cardano_database_v2_enabled == 'true' || steps.aggregator_capability_windows.outputs.cardano_database_v2_enabled == 'true'
shell: bash
working-directory: ./bin
run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --unstable --origin-tag CI cardano-db-v2 download $CARDANO_DATABASE_V2_SNAPSHOT_HASH ${{ matrix.extra_args }} 2>&1 | tee cdb-download-output.txt
run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --unstable --origin-tag CI cardano-db-v2 download $CARDANO_DATABASE_V2_SNAPSHOT_HASH ${{ matrix.extra_args }} 2>&1 | tee cdb-v2-download-output.txt

- name: Cardano Database V2 Snapshot / verify Cardano node starts successfully
if: runner.os == 'Linux' && steps.aggregator_capability_unix.outputs.cardano_database_v2_enabled == 'true'
Expand Down
4 changes: 2 additions & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion mithril-aggregator/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "mithril-aggregator"
version = "0.7.53"
version = "0.7.54"
description = "A Mithril Aggregator server"
authors = { workspace = true }
edition = { workspace = true }
Expand Down
5 changes: 3 additions & 2 deletions mithril-aggregator/src/artifact_builder/cardano_database.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ mod tests {
let ledger_file_size = 6666;
let volatile_file_size = 99;
let cardano_db = DummyCardanoDbBuilder::new("cdb-should_compute_valid_artifact")
.with_immutables(&[1, 2, 3])
.with_immutables(&[0, 1, 2, 3])
.append_immutable_trio()
.set_immutable_trio_file_size(immutable_trio_file_size)
.with_ledger_files(&["437"])
Expand Down Expand Up @@ -227,7 +227,8 @@ mod tests {
}

let immutable_artifact_builder = {
let number_of_immutable_file_loaded = beacon.immutable_file_number;
// We upload the immutable files in the [0,beacon.immutable_file_number] range
let number_of_immutable_file_loaded = beacon.immutable_file_number + 1;
let mut immutable_uploader = MockImmutableFilesUploader::new();
immutable_uploader
.expect_batch_upload()
Expand Down
21 changes: 16 additions & 5 deletions mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,8 +193,9 @@ impl ImmutableArtifactBuilder {
) -> StdResult<(Vec<PathBuf>, CompressionAlgorithm)> {
let mut archive_paths = vec![];
let compression_algorithm = self.snapshotter.compression_algorithm();
const FIRST_IMMUTABLE_FILE_NUMBER: ImmutableFileNumber = 0;

for immutable_file_number in 1..=up_to_immutable_file_number {
for immutable_file_number in FIRST_IMMUTABLE_FILE_NUMBER..=up_to_immutable_file_number {
let archive_name_without_extension = format!("{immutable_file_number:05}");
let archive_name = format!(
"{archive_name_without_extension}.{}",
Expand Down Expand Up @@ -352,7 +353,7 @@ mod tests {
let work_dir = get_builder_work_dir("upload_call_archive_creation_and_upload");
let test_dir = "upload_call_archive_creation_and_upload/cardano_database";
let cardano_db = DummyCardanoDbBuilder::new(test_dir)
.with_immutables(&[1, 2])
.with_immutables(&[0, 1, 2])
.build();

let db_directory = cardano_db.get_dir().to_path_buf();
Expand All @@ -368,6 +369,7 @@ mod tests {

let uploader = fake_uploader(
vec![
work_dir.join("00000.tar.gz").to_str().unwrap(),
work_dir.join("00001.tar.gz").to_str().unwrap(),
work_dir.join("00002.tar.gz").to_str().unwrap(),
],
Expand Down Expand Up @@ -444,7 +446,7 @@ mod tests {
let test_dir =
"snapshot_immutables_files_up_to_the_given_immutable_file_number/cardano_database";
let cardano_db = DummyCardanoDbBuilder::new(test_dir)
.with_immutables(&[1, 2])
.with_immutables(&[0, 1, 2])
.build();

let db_directory = cardano_db.get_dir().to_path_buf();
Expand Down Expand Up @@ -473,7 +475,11 @@ mod tests {

assert_equivalent(
archive_paths,
vec![work_dir.join("00001.tar.gz"), work_dir.join("00002.tar.gz")],
vec![
work_dir.join("00000.tar.gz"),
work_dir.join("00001.tar.gz"),
work_dir.join("00002.tar.gz"),
],
)
}

Expand Down Expand Up @@ -614,7 +620,7 @@ mod tests {
let work_dir = get_builder_work_dir("return_all_archives_but_not_rebuild_archives");
let test_dir = "return_all_archives_but_not_rebuild_archives/cardano_database";
let cardano_db = DummyCardanoDbBuilder::new(test_dir)
.with_immutables(&[1, 2, 3])
.with_immutables(&[0, 1, 2, 3])
.build();

let db_directory = cardano_db.get_dir().to_path_buf();
Expand All @@ -628,6 +634,7 @@ mod tests {
)
.unwrap();

create_fake_file(&work_dir.join("00000.tar.gz"), "00000 content");
create_fake_file(&work_dir.join("00001.tar.gz"), "00001 content");
create_fake_file(&work_dir.join("00002.tar.gz"), "00002 content");

Expand All @@ -647,12 +654,14 @@ mod tests {
assert_equivalent(
archive_paths,
vec![
work_dir.join("00000.tar.gz"),
work_dir.join("00001.tar.gz"),
work_dir.join("00002.tar.gz"),
work_dir.join("00003.tar.gz"),
],
);
// Check that the existing archives content have not changed
assert_file_content!(work_dir.join("00000.tar.gz"), "00000 content");
assert_file_content!(work_dir.join("00001.tar.gz"), "00001 content");
assert_file_content!(work_dir.join("00002.tar.gz"), "00002 content");
}
Expand All @@ -666,6 +675,7 @@ mod tests {
.expect_compression_algorithm()
.returning(|| CompressionAlgorithm::Gzip);

create_fake_file(&work_dir.join("00000.tar.gz"), "00000 content");
create_fake_file(&work_dir.join("00001.tar.gz"), "00001 content");
create_fake_file(&work_dir.join("00002.tar.gz"), "00002 content");
create_fake_file(&work_dir.join("00003.tar.gz"), "00003 content");
Expand All @@ -686,6 +696,7 @@ mod tests {
assert_equivalent(
archive_paths,
vec![
work_dir.join("00000.tar.gz"),
work_dir.join("00001.tar.gz"),
work_dir.join("00002.tar.gz"),
work_dir.join("00003.tar.gz"),
Expand Down
2 changes: 1 addition & 1 deletion mithril-client/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "mithril-client"
version = "0.12.7"
version = "0.12.8"
description = "Mithril client library"
authors = { workspace = true }
edition = { workspace = true }
Expand Down
9 changes: 3 additions & 6 deletions mithril-client/src/cardano_database_client/immutable_file_range.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,7 @@ impl ImmutableFileRange {
&self,
last_immutable_file_number: ImmutableFileNumber,
) -> StdResult<RangeInclusive<ImmutableFileNumber>> {
// The immutable file numbers start from 1 on all the networks except the 'devnet'
// when it is configured with aggressive protocol parameters for fast epochs (used in the e2e tests).
// We have taken the choice to consider that the file numbers start from 1 for all the networks.
const FIRST_IMMUTABLE_FILE_NUMBER: ImmutableFileNumber = 1;
const FIRST_IMMUTABLE_FILE_NUMBER: ImmutableFileNumber = 0;
let full_range = FIRST_IMMUTABLE_FILE_NUMBER..=last_immutable_file_number;

match self {
Expand Down Expand Up @@ -74,7 +71,7 @@ mod tests {
let result = immutable_file_range
.to_range_inclusive(last_immutable_file_number)
.unwrap();
assert_eq!(1..=10, result);
assert_eq!(0..=10, result);
}

#[test]
Expand Down Expand Up @@ -124,7 +121,7 @@ mod tests {
let result = immutable_file_range
.to_range_inclusive(last_immutable_file_number)
.unwrap();
assert_eq!(1..=8, result);
assert_eq!(0..=8, result);

let last_immutable_file_number = 7;
immutable_file_range
Expand Down
Loading