Handle multiple beacon types in Certificate

mithril-aggregator/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.4.50"
+version = "0.4.51"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-aggregator/src/artifact_builder/cardano_immutable_files_full.rs

@@ -13,8 +13,7 @@ use crate::{
 use super::ArtifactBuilder;
 use mithril_common::{
     entities::{
-        CardanoDbBeacon, Certificate, CompressionAlgorithm, ProtocolMessage,
-        ProtocolMessagePartKey, Snapshot,
+        CardanoDbBeacon, Certificate, CompressionAlgorithm, ProtocolMessagePartKey, Snapshot,
     },
     StdResult,
 };
@@ -55,16 +54,11 @@ impl CardanoImmutableFilesFullArtifactBuilder {
     async fn create_snapshot_archive(
         &self,
         beacon: &CardanoDbBeacon,
-        protocol_message: &ProtocolMessage,
+        snapshot_digest: &str,
     ) -> StdResult<OngoingSnapshot> {
         debug!("CardanoImmutableFilesFullArtifactBuilder: create snapshot archive");
 
         let snapshotter = self.snapshotter.clone();
-        let snapshot_digest = protocol_message
-            .get_message_part(&ProtocolMessagePartKey::SnapshotDigest)
-            .ok_or_else(|| {
-                CardanoImmutableFilesFullArtifactError::MissingProtocolMessage(beacon.clone())
-            })?;
         let snapshot_name = format!(
             "{}-e{}-i{}.{}.{}",
             beacon.network,
@@ -107,23 +101,16 @@ impl CardanoImmutableFilesFullArtifactBuilder {
 
     async fn create_snapshot(
         &self,
-        certificate: &Certificate,
+        beacon: CardanoDbBeacon,
         ongoing_snapshot: &OngoingSnapshot,
+        snapshot_digest: String,
         remote_locations: Vec<String>,
     ) -> StdResult<Snapshot> {
         debug!("CardanoImmutableFilesFullArtifactBuilder: create snapshot");
-        let snapshot_digest = certificate
-            .protocol_message
-            .get_message_part(&ProtocolMessagePartKey::SnapshotDigest)
-            .ok_or_else(|| {
-                CardanoImmutableFilesFullArtifactError::MissingProtocolMessage(
-                    certificate.beacon.clone(),
-                )
-            })?
-            .to_owned();
+
         let snapshot = Snapshot::new(
             snapshot_digest,
-            certificate.beacon.to_owned(),
+            beacon,
             *ongoing_snapshot.get_file_size(),
             remote_locations,
             self.compression_algorithm,
@@ -141,8 +128,16 @@ impl ArtifactBuilder<CardanoDbBeacon, Snapshot> for CardanoImmutableFilesFullArt
         beacon: CardanoDbBeacon,
         certificate: &Certificate,
     ) -> StdResult<Snapshot> {
+        let snapshot_digest = certificate
+            .protocol_message
+            .get_message_part(&ProtocolMessagePartKey::SnapshotDigest)
+            .ok_or_else(|| {
+                CardanoImmutableFilesFullArtifactError::MissingProtocolMessage(beacon.clone())
+            })?
+            .to_owned();
+
         let ongoing_snapshot = self
-            .create_snapshot_archive(&beacon, &certificate.protocol_message)
+            .create_snapshot_archive(&beacon, &snapshot_digest)
             .await
             .with_context(|| {
                 "Cardano Immutable Files Full Artifact Builder can not create snapshot archive"
@@ -155,7 +150,7 @@ impl ArtifactBuilder<CardanoDbBeacon, Snapshot> for CardanoImmutableFilesFullArt
             })?;
 
         let snapshot = self
-            .create_snapshot(certificate, &ongoing_snapshot, locations)
+            .create_snapshot(beacon, &ongoing_snapshot, snapshot_digest, locations)
             .await?;
 
         Ok(snapshot)
@@ -194,7 +189,7 @@ mod tests {
                 CompressionAlgorithm::Zstandard,
             );
         let artifact = cardano_immutable_files_full_artifact_builder
-            .compute_artifact(beacon, &certificate)
+            .compute_artifact(beacon.clone(), &certificate)
             .await
             .unwrap();
         let last_ongoing_snapshot = dumb_snapshotter
@@ -208,7 +203,7 @@ mod tests {
             .expect("A snapshot should have been 'uploaded'")];
         let artifact_expected = Snapshot::new(
             snapshot_digest.to_owned(),
-            certificate.beacon.to_owned(),
+            beacon,
             *last_ongoing_snapshot.get_file_size(),
             remote_locations,
             CompressionAlgorithm::Zstandard,
@@ -245,11 +240,7 @@ mod tests {
     #[tokio::test]
     async fn snapshot_archive_name_after_beacon_values() {
         let beacon = CardanoDbBeacon::new("network".to_string(), 20, 145);
-        let mut message = ProtocolMessage::new();
-        message.set_message_part(
-            ProtocolMessagePartKey::SnapshotDigest,
-            "test+digest".to_string(),
-        );
+        let digest = "test+digest";
 
         let cardano_immutable_files_full_artifact_builder =
             CardanoImmutableFilesFullArtifactBuilder::new(
@@ -260,27 +251,21 @@ mod tests {
             );
 
         let ongoing_snapshot = cardano_immutable_files_full_artifact_builder
-            .create_snapshot_archive(&beacon, &message)
+            .create_snapshot_archive(&beacon, digest)
             .await
             .expect("create_snapshot_archive should not fail");
 
         assert_eq!(
             Path::new(&format!(
-                "{}-e{}-i{}.{}.tar.gz",
-                beacon.network, *beacon.epoch, beacon.immutable_file_number, "test+digest"
+                "{}-e{}-i{}.{digest}.tar.gz",
+                beacon.network, *beacon.epoch, beacon.immutable_file_number,
             )),
             ongoing_snapshot.get_file_path()
         );
     }
 
     #[tokio::test]
     async fn snapshot_archive_name_after_compression_algorithm() {
-        let mut message = ProtocolMessage::new();
-        message.set_message_part(
-            ProtocolMessagePartKey::SnapshotDigest,
-            "test+digest".to_string(),
-        );
-
         let mut invalid_result: Vec<CompressionAlgorithm> = vec![];
 
         for algorithm in CompressionAlgorithm::list() {
@@ -293,7 +278,7 @@ mod tests {
                 );
 
             let ongoing_snapshot = cardano_immutable_files_full_artifact_builder
-                .create_snapshot_archive(&CardanoDbBeacon::default(), &message)
+                .create_snapshot_archive(&CardanoDbBeacon::default(), "test+digest")
                 .await
                 .expect("create_snapshot_archive should not fail");
             let file_name = ongoing_snapshot

mithril-aggregator/src/artifact_builder/cardano_transactions.rs

@@ -71,13 +71,13 @@ mod tests {
             certificate
         };
 
+        let beacon = certificate.as_cardano_db_beacon();
         let cardano_transaction_artifact_builder = CardanoTransactionsArtifactBuilder::new();
         let artifact = cardano_transaction_artifact_builder
-            .compute_artifact(certificate.beacon.clone(), &certificate)
+            .compute_artifact(beacon.clone(), &certificate)
             .await
             .unwrap();
-        let artifact_expected =
-            CardanoTransactionsSnapshot::new("merkleroot".to_string(), certificate.beacon);
+        let artifact_expected = CardanoTransactionsSnapshot::new("merkleroot".to_string(), beacon);
         assert_eq!(artifact_expected, artifact);
     }
 

mithril-aggregator/src/configuration.rs

@@ -9,8 +9,8 @@ use std::path::PathBuf;
 use std::str::FromStr;
 
 use mithril_common::entities::{
-    CardanoDbBeacon, CompressionAlgorithm, HexEncodedGenesisVerificationKey, ProtocolParameters,
-    SignedEntityType, SignedEntityTypeDiscriminants,
+    CompressionAlgorithm, HexEncodedGenesisVerificationKey, ProtocolParameters, SignedEntityType,
+    SignedEntityTypeDiscriminants, TimePoint,
 };
 use mithril_common::{CardanoNetwork, StdResult};
 
@@ -285,12 +285,14 @@ impl Configuration {
     /// The signed entity types are discarded if they are not declared in the [SignedEntityType] enum.
     pub fn list_allowed_signed_entity_types(
         &self,
-        beacon: &CardanoDbBeacon,
+        time_point: &TimePoint,
     ) -> StdResult<Vec<SignedEntityType>> {
         let allowed_discriminants = self.list_allowed_signed_entity_types_discriminants()?;
         let signed_entity_types = allowed_discriminants
             .into_iter()
-            .map(|discriminant| SignedEntityType::from_beacon(&discriminant, beacon))
+            .map(|discriminant| {
+                SignedEntityType::from_time_point(&discriminant, &self.network, time_point)
+            })
             .collect();
 
         Ok(signed_entity_types)
@@ -502,16 +504,20 @@ mod test {
 
     #[test]
     fn test_list_allowed_signed_entity_types_without_specific_configuration() {
+        let beacon = fake_data::beacon();
         let config = Configuration {
             signed_entity_types: None,
+            network: beacon.network.clone(),
             ..Configuration::new_sample()
         };
-        let beacon = fake_data::beacon();
+        let time_point = TimePoint::new(*beacon.epoch, beacon.immutable_file_number);
 
         let discriminants = config
             .list_allowed_signed_entity_types_discriminants()
             .unwrap();
-        let signed_entity_types = config.list_allowed_signed_entity_types(&beacon).unwrap();
+        let signed_entity_types = config
+            .list_allowed_signed_entity_types(&time_point)
+            .unwrap();
 
         assert_eq!(
             vec![
@@ -531,18 +537,22 @@ mod test {
 
     #[test]
     fn test_list_allowed_signed_entity_types_with_specific_configuration() {
+        let beacon = fake_data::beacon();
         let config = Configuration {
+            network: beacon.network.clone(),
             signed_entity_types: Some(
                 "MithrilStakeDistribution,Unknown, CardanoStakeDistribution".to_string(),
             ),
             ..Configuration::new_sample()
         };
-        let beacon = fake_data::beacon();
+        let time_point = TimePoint::new(*beacon.epoch, beacon.immutable_file_number);
 
         let discriminants = config
             .list_allowed_signed_entity_types_discriminants()
             .unwrap();
-        let signed_entity_types = config.list_allowed_signed_entity_types(&beacon).unwrap();
+        let signed_entity_types = config
+            .list_allowed_signed_entity_types(&time_point)
+            .unwrap();
 
         assert_eq!(
             vec![

mithril-aggregator/src/database/migration.rs

@@ -619,5 +619,85 @@ insert into signed_entity_type (signed_entity_type_id, name)
     values  (3, 'Cardano Transactions');
 "#,
         ),
+        // Migration 22
+        // Certificate table:
+        // * Remove beacon
+        // * Add network, immutable file number, signed_entity_type_id, signed_entity_type columns
+        SqlMigration::new(
+            22,
+            r#"
+-- disable foreign keys since we will delete tables linked using them
+pragma foreign_keys=false;
+
+CREATE TABLE IF NOT EXISTS "new_certificate" (
+    certificate_id              text     not null,
+    parent_certificate_id       text,
+    message                     text     not null,
+    signature                   text     not null,
+    aggregate_verification_key  text     not null,
+    epoch                       integer  not null,
+    network                     text     not null,
+    immutable_file_number       integer  not null,
+    signed_entity_type_id       integer  not null,
+    signed_entity_beacon        json     not null,
+    protocol_version            text     not null,
+    protocol_parameters         json     not null,
+    protocol_message            json     not null,
+    signers                     json     not null,
+    initiated_at                text     not null,
+    sealed_at                   text     not null,
+    primary key (certificate_id),
+    foreign key (parent_certificate_id) references certificate(certificate_id)
+    foreign key (signed_entity_type_id) references signed_entity_type(signed_entity_type_id)
+);
+
+insert into new_certificate
+        ( certificate_id, parent_certificate_id, message, signature, aggregate_verification_key,
+        epoch,
+        network,
+        immutable_file_number,
+        signed_entity_type_id,
+        signed_entity_beacon,
+        protocol_version, protocol_parameters, protocol_message,
+        signers, initiated_at, sealed_at)
+    select c.certificate_id, c.parent_certificate_id, c.message, c.signature, c.aggregate_verification_key,
+        c.epoch,
+        json_extract(c.beacon, '$.network'),
+        json_extract(c.beacon, '$.immutable_file_number'),
+        -- genesis certificate doesn't have a signed_entity, we can just use directly the MithrilStakeDistribution
+        coalesce(s.signed_entity_type_id, 0),
+        -- genesis certificate doesn't have a signed_entity, so we need to deduce it from the old certificate
+        coalesce(s.beacon, c.epoch),
+        c.protocol_version, c.protocol_parameters, c.protocol_message,
+        c.signers, c.initiated_at, c.sealed_at
+    from certificate c
+        left join signed_entity s on s.certificate_id = c.certificate_id
+    order by c.rowid asc;
+
+
+drop table certificate;
+alter table new_certificate rename to certificate;
+
+CREATE INDEX epoch_index on certificate(epoch);
+
+-- reenable foreign keys
+pragma foreign_key_check;
+pragma foreign_keys=true;
+"#,
+        ),
+        // Migration 23
+        // Alter `pending_certificate` table to use only an Epoch instead of a full beacon.
+        SqlMigration::new(
+            23,
+            r#"
+create table if not exists pending_certificate (key_hash text primary key, key json not null, value json not null);
+update pending_certificate
+    set value = 
+        json_remove(
+            json_insert(value, '$.epoch', json_extract(value, '$.beacon.epoch')),
+            '$.beacon'
+        );
+        "#,
+        ),
     ]
 }