diff --git a/avro-schema/validation-batch.avsc b/avro-schema/validation-batch.avsc new file mode 100644 index 000000000..4c22e4525 --- /dev/null +++ b/avro-schema/validation-batch.avsc @@ -0,0 +1,22 @@ +{ + "namespace": "org.abetterinternet.prio.v1", + "type": "record", + "name": "PrioValidityBatch", + "fields": [ + { + "name": "sig", + "type": "org.abetterinternet.prio.v1.PrioBatchSignature", + "doc": "A signature over the header field." + }, + { + "name": "header", + "type": "bytes", + "doc": "A serialized PrioValidityHeader for the batch." + }, + { + "name": "packets", + "type": "bytes", + "doc": "A serialized sequence of PrioValidityPackets making up the batch." + } + ] +} diff --git a/facilitator/Cargo.lock b/facilitator/Cargo.lock index ddf482978..4d9f76f39 100644 --- a/facilitator/Cargo.lock +++ b/facilitator/Cargo.lock @@ -627,6 +627,7 @@ dependencies = [ "k8s-openapi", "kube", "kube-runtime", + "lazy_static", "mockito", "p256", "pem 1.0.1", diff --git a/facilitator/Cargo.toml b/facilitator/Cargo.toml index d864d61e7..95ff66064 100644 --- a/facilitator/Cargo.toml +++ b/facilitator/Cargo.toml @@ -26,6 +26,7 @@ jsonwebtoken = "7" k8s-openapi = { version = "0.12.0", default-features = false, features = ["v1_20"] } kube = "0.57.0" kube-runtime = "0.57.0" +lazy_static = "1" p256 = "0.9.0" pem = "1.0" pkix = "0.1.1" diff --git a/facilitator/src/batch.rs b/facilitator/src/batch.rs index 165a04755..3f8da28ce 100644 --- a/facilitator/src/batch.rs +++ b/facilitator/src/batch.rs @@ -6,7 +6,7 @@ use crate::{ DigestWriter, DATE_FORMAT, }; use anyhow::{anyhow, Context, Result}; -use avro_rs::{Reader, Schema, Writer}; +use avro_rs::{Reader, Writer}; use chrono::NaiveDateTime; use ring::{ digest::Digest, @@ -112,7 +112,6 @@ pub struct BatchReader<'a, H, P> { trace_id: &'a Uuid, batch: Batch, transport: &'a mut dyn Transport, - packet_schema: Schema, permit_malformed_batch: bool, metrics_collector: Option<&'a BatchReaderMetricsCollector>, logger: Logger, @@ -142,7 +141,6 @@ impl<'a, H: Header, P: Packet> BatchReader<'a, H, P> { trace_id, batch, transport, - packet_schema: P::schema(), permit_malformed_batch, metrics_collector: None, logger, @@ -254,7 +252,7 @@ impl<'a, H: Header, P: Packet> BatchReader<'a, H, P> { } // ... then return a packet reader. - Reader::with_schema(&self.packet_schema, Cursor::new(entire_packet_file)) + Reader::with_schema(P::schema(), Cursor::new(entire_packet_file)) .context("failed to create Avro reader for packets") } } @@ -265,7 +263,6 @@ impl<'a, H: Header, P: Packet> BatchReader<'a, H, P> { pub struct BatchWriter<'a, H, P> { batch: Batch, transport: &'a mut dyn Transport, - packet_schema: Schema, trace_id: &'a Uuid, phantom_header: PhantomData<*const H>, phantom_packet: PhantomData<*const P>, @@ -276,7 +273,6 @@ impl<'a, H: Header, P: Packet> BatchWriter<'a, H, P> { BatchWriter { batch, transport, - packet_schema: P::schema(), trace_id, phantom_header: PhantomData, phantom_packet: PhantomData, @@ -320,7 +316,7 @@ impl<'a, H: Header, P: Packet> BatchWriter<'a, H, P> { .transport .put(self.batch.packet_file_key(), self.trace_id)?; let mut digest_writer = DigestWriter::new(&mut transport_writer); - let mut writer = Writer::new(&self.packet_schema, &mut digest_writer); + let mut writer = Writer::new(P::schema(), &mut digest_writer); let result = operation(&mut writer); writer diff --git a/facilitator/src/idl.rs b/facilitator/src/idl.rs index 568604c63..c8a3d76d3 100644 --- a/facilitator/src/idl.rs +++ b/facilitator/src/idl.rs @@ -5,6 +5,7 @@ use avro_rs::{ types::{Record, Value}, Reader, Schema, Writer, }; +use lazy_static::lazy_static; use prio::{ field::FieldPriov2, server::{Server, ServerError, VerificationMessage}, @@ -17,14 +18,47 @@ use std::{ }; use uuid::Uuid; -const BATCH_SIGNATURE_SCHEMA: &str = include_str!("../../avro-schema/batch-signature.avsc"); -const INGESTION_HEADER_SCHEMA: &str = include_str!("../../avro-schema/ingestion-header.avsc"); -const INGESTION_DATA_SHARE_PACKET_SCHEMA: &str = - include_str!("../../avro-schema/ingestion-data-share-packet.avsc"); -const VALIDATION_HEADER_SCHEMA: &str = include_str!("../../avro-schema/validation-header.avsc"); -const VALIDATION_PACKET_SCHEMA: &str = include_str!("../../avro-schema/validation-packet.avsc"); -const SUM_PART_SCHEMA: &str = include_str!("../../avro-schema/sum-part.avsc"); -const INVALID_PACKET_SCHEMA: &str = include_str!("../../avro-schema/invalid-packet.avsc"); +lazy_static! { + // Schema::parse_list returns its schemas in the order they are provided; + // for ease-of-use, we pull these out into their own helpfully-named + // variables. `_SCHEMAS` should be referenced only within this + // `lazy_static` block (and ideally would not exist at all). Schema users + // should refer to the various `&'static Schema` exported below. + static ref _SCHEMAS: Vec = { + const BATCH_SIGNATURE_SCHEMA: &str = include_str!("../../avro-schema/batch-signature.avsc"); + const INGESTION_HEADER_SCHEMA: &str = + include_str!("../../avro-schema/ingestion-header.avsc"); + const INGESTION_DATA_SHARE_PACKET_SCHEMA: &str = + include_str!("../../avro-schema/ingestion-data-share-packet.avsc"); + const VALIDATION_HEADER_SCHEMA: &str = + include_str!("../../avro-schema/validation-header.avsc"); + const VALIDATION_PACKET_SCHEMA: &str = + include_str!("../../avro-schema/validation-packet.avsc"); + const SUM_PART_SCHEMA: &str = include_str!("../../avro-schema/sum-part.avsc"); + const INVALID_PACKET_SCHEMA: &str = include_str!("../../avro-schema/invalid-packet.avsc"); + const VALIDATION_BATCH_SCHEMA: &str = include_str!("../../avro-schema/validation-batch.avsc"); + + Schema::parse_list(&[ + BATCH_SIGNATURE_SCHEMA, + INGESTION_HEADER_SCHEMA, + INGESTION_DATA_SHARE_PACKET_SCHEMA, + VALIDATION_HEADER_SCHEMA, + VALIDATION_PACKET_SCHEMA, + SUM_PART_SCHEMA, + INVALID_PACKET_SCHEMA, + VALIDATION_BATCH_SCHEMA, + ]) + .unwrap() + }; + static ref BATCH_SIGNATURE_SCHEMA: &'static Schema = &_SCHEMAS[0]; + static ref INGESTION_HEADER_SCHEMA: &'static Schema = &_SCHEMAS[1]; + static ref INGESTION_DATA_SHARE_PACKET_SCHEMA: &'static Schema = &_SCHEMAS[2]; + static ref VALIDATION_HEADER_SCHEMA: &'static Schema = &_SCHEMAS[3]; + static ref VALIDATION_PACKET_SCHEMA: &'static Schema = &_SCHEMAS[4]; + static ref SUM_PART_SCHEMA: &'static Schema = &_SCHEMAS[5]; + static ref INVALID_PACKET_SCHEMA: &'static Schema = &_SCHEMAS[6]; + static ref VALIDATION_BATCH_SCHEMA: &'static Schema = &_SCHEMAS[7]; +} pub trait Header: Sized { /// Returns the SHA256 digest of the packet file this header describes. @@ -41,7 +75,7 @@ pub trait Packet: Sized { /// that unlike other structures, this does not take a primitive /// std::io::Read, because we do not want to create a new Avro schema and /// reader for each packet. The Reader must have been created with the - //// schema returned from Packet::schema. + /// schema returned from Packet::schema. fn read(reader: &mut Reader) -> Result; /// Serializes and writes a single Packet to the provided avro_rs::Writer. @@ -51,17 +85,9 @@ pub trait Packet: Sized { /// schema returned from Packet::schema. fn write(&self, writer: &mut Writer) -> Result<(), Error>; - /// Implementations of Packet should return their Avro schemas as strings - /// from this method. - fn schema_raw() -> &'static str; - - /// Creates an avro_rs::Schema from the packet schema. For constructing the - /// avro_rs::{Reader, Writer} to use in Packet::{read, write}. Since this - /// only ever uses a schema whose correctness we can guarantee, it panics on - /// failure. - fn schema() -> Schema { - Schema::parse_str(Self::schema_raw()).unwrap() - } + /// Provides the avro_rs::Schema used by the packet. For constructing the + /// avro_rs::{Reader, Writer} to use in Packet::{read, write}. + fn schema() -> &'static Schema; } /// The file containing signatures over the ingestion batch header and packet @@ -76,20 +102,12 @@ impl BatchSignature { /// Reads and parses one BatchSignature from the provided std::io::Read /// instance. pub fn read(reader: R) -> Result { - let schema = Schema::parse_str(BATCH_SIGNATURE_SCHEMA).map_err(|e| { - Error::AvroError("failed to parse ingestion signature schema".to_owned(), e) - })?; - let mut reader = Reader::with_schema(&schema, reader) + let mut reader = Reader::with_schema(*BATCH_SIGNATURE_SCHEMA, reader) .map_err(|e| Error::AvroError("failed to create Avro reader".to_owned(), e))?; // We expect exactly one record and for it to be an ingestion signature - let record = match reader.next() { - Some(Ok(Value::Record(r))) => r, - Some(Ok(_)) => { - return Err(Error::MalformedHeaderError( - "value is not a record".to_owned(), - )) - } + let value = match reader.next() { + Some(Ok(value)) => value, Some(Err(e)) => { return Err(Error::AvroError( "failed to read record from Avro reader".to_owned(), @@ -104,6 +122,38 @@ impl BatchSignature { )); } + BatchSignature::try_from(value) + } + + /// Serializes this signature into Avro format and writes it to the provided + /// std::io::Write instance. + pub fn write(&self, writer: &mut W) -> Result<(), Error> { + let mut writer = Writer::new(*BATCH_SIGNATURE_SCHEMA, writer); + writer.append(Value::from(self)).map_err(|e| { + Error::AvroError("failed to append record to Avro writer".to_owned(), e) + })?; + + writer + .flush() + .map_err(|e| Error::AvroError("failed to flush Avro writer".to_owned(), e))?; + + Ok(()) + } +} + +impl TryFrom for BatchSignature { + type Error = Error; + + fn try_from(value: Value) -> Result { + let fields = match value { + Value::Record(fields) => fields, + _ => { + return Err(Error::MalformedBatchError( + "value is not a Record".to_owned(), + )) + } + }; + // Here we might wish to use from_value::(record) but // avro_rs does not seem to recognize it as a Bytes and fails to // deserialize it. The value we unwrapped from reader.next above is a @@ -112,65 +162,38 @@ impl BatchSignature { let mut batch_header_signature = None; let mut key_identifier = None; - for tuple in record { - match (tuple.0.as_str(), tuple.1) { + for (field_name, field_value) in fields { + match (field_name.as_str(), field_value) { ("batch_header_signature", Value::Bytes(v)) => batch_header_signature = Some(v), ("key_identifier", Value::String(v)) => key_identifier = Some(v), - (f, _) => { - return Err(Error::MalformedHeaderError(format!( - "unexpected field {} in record", - f - ))) - } + _ => (), } } - if batch_header_signature.is_none() || key_identifier.is_none() { - return Err(Error::MalformedHeaderError( - "missing fields in record".to_owned(), - )); - } - Ok(BatchSignature { - batch_header_signature: batch_header_signature.unwrap(), - key_identifier: key_identifier.unwrap(), + batch_header_signature: batch_header_signature.ok_or_else(|| { + Error::MalformedHeaderError("missing batch_header_signature".to_owned()) + })?, + key_identifier: key_identifier + .ok_or_else(|| Error::MalformedHeaderError("missing key_identifier".to_owned()))?, }) } +} - /// Serializes this signature into Avro format and writes it to the provided - /// std::io::Write instance. - pub fn write(&self, writer: &mut W) -> Result<(), Error> { - let schema = Schema::parse_str(BATCH_SIGNATURE_SCHEMA).map_err(|e| { - Error::AvroError("failed to parse ingestion signature schema".to_owned(), e) - })?; - let mut writer = Writer::new(&schema, writer); - - let mut record = match Record::new(writer.schema()) { - Some(r) => r, - None => { - // avro_rs docs say this can only happen "if the `Schema is not - // a `Schema::Record` variant", which shouldn't ever happen, so - // panic for debugging - // https://docs.rs/avro-rs/0.11.0/avro_rs/types/struct.Record.html#method.new - panic!("Unable to create Record from ingestion signature schema"); - } - }; - +impl From<&BatchSignature> for Value { + fn from(sig: &BatchSignature) -> Value { + // avro_rs docs say this can only happen "if the `Schema is not + // a `Schema::Record` variant", which shouldn't ever happen, so + // panic for debugging + // https://docs.rs/avro-rs/0.11.0/avro_rs/types/struct.Record.html#method.new + let mut record = Record::new(*BATCH_SIGNATURE_SCHEMA) + .expect("Unable to create record from ingestion signature schema"); record.put( "batch_header_signature", - Value::Bytes(self.batch_header_signature.clone()), + Value::Bytes(sig.batch_header_signature.clone()), ); - record.put("key_identifier", Value::String(self.key_identifier.clone())); - - writer.append(record).map_err(|e| { - Error::AvroError("failed to append record to Avro writer".to_owned(), e) - })?; - - writer - .flush() - .map_err(|e| Error::AvroError("failed to flush Avro writer".to_owned(), e))?; - - Ok(()) + record.put("key_identifier", Value::String(sig.key_identifier.clone())); + Value::Record(record.fields) } } @@ -208,10 +231,7 @@ impl Header for IngestionHeader { } fn read(reader: R) -> Result { - let schema = Schema::parse_str(INGESTION_HEADER_SCHEMA).map_err(|e| { - Error::AvroError("failed to parse ingestion header schema".to_owned(), e) - })?; - let mut reader = Reader::with_schema(&schema, reader).map_err(|e| { + let mut reader = Reader::with_schema(*INGESTION_HEADER_SCHEMA, reader).map_err(|e| { Error::AvroError("failed to create reader for ingestion header".to_owned(), e) })?; @@ -314,10 +334,7 @@ impl Header for IngestionHeader { } fn write(&self, writer: &mut W) -> Result<(), Error> { - let schema = Schema::parse_str(INGESTION_HEADER_SCHEMA).map_err(|e| { - Error::AvroError("failed to parse ingestion header schema".to_owned(), e) - })?; - let mut writer = Writer::new(&schema, writer); + let mut writer = Writer::new(*INGESTION_HEADER_SCHEMA, writer); // Ideally we would just do `writer.append_ser(self)` to use Serde serialization to write // the record but there seems to be some problem with serializing UUIDs, so we have to @@ -381,8 +398,8 @@ pub struct IngestionDataSharePacket { } impl Packet for IngestionDataSharePacket { - fn schema_raw() -> &'static str { - INGESTION_DATA_SHARE_PACKET_SCHEMA + fn schema() -> &'static Schema { + *INGESTION_DATA_SHARE_PACKET_SCHEMA } fn read(reader: &mut Reader) -> Result { @@ -589,10 +606,7 @@ impl Header for ValidationHeader { } fn read(reader: R) -> Result { - let schema = Schema::parse_str(VALIDATION_HEADER_SCHEMA).map_err(|e| { - Error::AvroError("failed to parse validation header schema".to_owned(), e) - })?; - let mut reader = Reader::with_schema(&schema, reader).map_err(|e| { + let mut reader = Reader::with_schema(*VALIDATION_HEADER_SCHEMA, reader).map_err(|e| { Error::AvroError( "failed to create reader for validation header".to_owned(), e, @@ -690,10 +704,7 @@ impl Header for ValidationHeader { } fn write(&self, writer: &mut W) -> Result<(), Error> { - let schema = Schema::parse_str(VALIDATION_HEADER_SCHEMA).map_err(|e| { - Error::AvroError("failed to parse validation header schema".to_owned(), e) - })?; - let mut writer = Writer::new(&schema, writer); + let mut writer = Writer::new(*VALIDATION_HEADER_SCHEMA, writer); let mut record = match Record::new(writer.schema()) { Some(r) => r, @@ -740,8 +751,8 @@ pub struct ValidationPacket { } impl Packet for ValidationPacket { - fn schema_raw() -> &'static str { - VALIDATION_PACKET_SCHEMA + fn schema() -> &'static Schema { + *VALIDATION_PACKET_SCHEMA } fn read(reader: &mut Reader) -> Result { @@ -830,9 +841,7 @@ impl Header for SumPart { } fn read(reader: R) -> Result { - let schema = Schema::parse_str(SUM_PART_SCHEMA) - .map_err(|e| Error::AvroError("failed to parse sum part schema".to_owned(), e))?; - let mut reader = Reader::with_schema(&schema, reader) + let mut reader = Reader::with_schema(*SUM_PART_SCHEMA, reader) .map_err(|e| Error::AvroError("failed to create reader for sum part".to_owned(), e))?; // We expect exactly one record in the reader and for it to be a sum @@ -974,9 +983,7 @@ impl Header for SumPart { } fn write(&self, writer: &mut W) -> Result<(), Error> { - let schema = Schema::parse_str(SUM_PART_SCHEMA) - .map_err(|e| Error::AvroError("failed to parse sum part schema".to_owned(), e))?; - let mut writer = Writer::new(&schema, writer); + let mut writer = Writer::new(*SUM_PART_SCHEMA, writer); // Ideally we would just do `writer.append_ser(self)` to use Serde serialization to write // the record but there seems to be some problem with serializing UUIDs, so we have to @@ -1043,8 +1050,8 @@ pub struct InvalidPacket { } impl Packet for InvalidPacket { - fn schema_raw() -> &'static str { - INVALID_PACKET_SCHEMA + fn schema() -> &'static Schema { + *INVALID_PACKET_SCHEMA } fn read(reader: &mut Reader) -> Result { @@ -1087,35 +1094,139 @@ impl Packet for InvalidPacket { } } +#[derive(Debug, Serialize, Deserialize, PartialEq)] +pub struct ValidationBatch { + pub sig: BatchSignature, + pub header: Vec, + pub packets: Vec, +} + +impl ValidationBatch { + /// Reads and parses one Header from the provided std::io::Read instance. + pub fn read(reader: R) -> Result { + let mut reader = Reader::with_schema(*VALIDATION_BATCH_SCHEMA, reader) + .map_err(|e| Error::AvroError("failed to create Avro reader".to_owned(), e))?; + // We expect exactly one record and for it to be an ingestion signature + let value = match reader.next() { + Some(Ok(value)) => value, + Some(Err(e)) => { + return Err(Error::AvroError( + "failed to read record from Avro reader".to_owned(), + e, + )); + } + None => return Err(Error::EofError), + }; + if reader.next().is_some() { + return Err(Error::MalformedBatchError( + "excess value in reader".to_owned(), + )); + } + + ValidationBatch::try_from(value) + } + + /// Serializes this message into Avro format and writes it to the provided + /// std::io::Write instance. + pub fn write(&self, writer: &mut W) -> Result<(), Error> { + let mut writer = Writer::new(*VALIDATION_BATCH_SCHEMA, writer); + writer.append(Value::from(self)).map_err(|e| { + Error::AvroError("failed to append record to Avro writer".to_owned(), e) + })?; + writer + .flush() + .map_err(|e| Error::AvroError("failed to flush".to_owned(), e))?; + Ok(()) + } +} + +impl TryFrom for ValidationBatch { + type Error = Error; + + fn try_from(value: Value) -> Result { + let fields = match value { + Value::Record(fields) => fields, + _ => { + return Err(Error::MalformedBatchError( + "value is not a Record".to_owned(), + )) + } + }; + + let mut sig = None; + let mut header = None; + let mut packets = None; + + for (field_name, field_value) in fields { + match (field_name.as_str(), field_value) { + ("sig", field_value) => sig = Some(BatchSignature::try_from(field_value)?), + ("header", Value::Bytes(buf)) => header = Some(buf), + ("packets", Value::Bytes(buf)) => packets = Some(buf), + _ => (), + } + } + + Ok(ValidationBatch { + sig: sig.ok_or_else(|| Error::MalformedBatchError("missing sig".to_owned()))?, + header: header + .ok_or_else(|| Error::MalformedBatchError("missing header".to_owned()))?, + packets: packets + .ok_or_else(|| Error::MalformedBatchError("missing packets".to_owned()))?, + }) + } +} + +impl From<&ValidationBatch> for Value { + fn from(batch: &ValidationBatch) -> Self { + let mut record = Record::new(*VALIDATION_BATCH_SCHEMA) + .expect("Unable to create record from validation batch schema"); + record.put("sig", Value::from(&batch.sig)); + record.put("header", Value::Bytes(batch.header.clone())); + record.put("packets", Value::Bytes(batch.packets.clone())); + Value::Record(record.fields) + } +} + #[cfg(test)] mod tests { use super::*; use assert_matches::assert_matches; - #[test] - fn roundtrip_batch_signature() { - let signature1 = BatchSignature { - batch_header_signature: vec![1u8, 2u8, 3u8, 4u8], - key_identifier: "my-cool-key".to_owned(), - }; - let signature2 = BatchSignature { - batch_header_signature: vec![5u8, 6u8, 7u8, 9u8], - key_identifier: "my-other-key".to_owned(), - }; + lazy_static! { + static ref GOLDEN_BATCH_SIGNATURES: Vec<(BatchSignature, Vec)> = { + let batch_signature_bytes: Vec = hex::decode("4f626a0104166176726f2e736368656d61fa027b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f42617463685369676e6174757265222c226669656c6473223a5b7b226e616d65223a2262617463685f6865616465725f7369676e6174757265222c2274797065223a226279746573227d2c7b226e616d65223a226b65795f6964656e746966696572222c2274797065223a22737472696e67227d5d7d146176726f2e636f646563086e756c6c00d0c5ed8df0137655e2aa2054a0c8c62902220801020304166d792d636f6f6c2d6b6579d0c5ed8df0137655e2aa2054a0c8c629").unwrap(); + let batch_signature: BatchSignature = BatchSignature { + batch_header_signature: vec![1u8, 2u8, 3u8, 4u8], + key_identifier: "my-cool-key".to_owned(), + }; - let mut record_vec = Vec::new(); + vec![(batch_signature, batch_signature_bytes)] + }; + } - signature1.write(&mut record_vec).unwrap(); - let signature_again = BatchSignature::read(&record_vec[..]).unwrap(); - assert_eq!(signature1, signature_again); - assert!(signature2 != signature_again); + #[test] + fn read_batch_signature() { + for (want_signature, signature_bytes) in &*GOLDEN_BATCH_SIGNATURES { + let batch_signature = BatchSignature::read(&signature_bytes[..]).unwrap(); + assert_eq!(want_signature, &batch_signature) + } } #[test] - fn roundtrip_ingestion_header() { - let headers = &[ - IngestionHeader { - batch_uuid: Uuid::new_v4(), + fn roundtrip_batch_signature() { + for (signature, _) in &*GOLDEN_BATCH_SIGNATURES { + let mut record_vec = Vec::new(); + signature.write(&mut record_vec).unwrap(); + let signature_again = BatchSignature::read(&record_vec[..]).unwrap(); + assert_eq!(signature, &signature_again); + } + } + + lazy_static! { + static ref GOLDEN_INGESTION_HEADERS: Vec<(IngestionHeader, Vec)> = { + let ingestion_header_1_bytes = hex::decode("4f626a0104146176726f2e636f646563086e756c6c166176726f2e736368656d61e8097b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f496e67657374696f6e486561646572222c226669656c6473223a5b7b226e616d65223a2262617463685f75756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d2c7b226e616d65223a226e616d65222c2274797065223a22737472696e67227d2c7b226e616d65223a2262696e73222c2274797065223a22696e74227d2c7b226e616d65223a22657073696c6f6e222c2274797065223a22646f75626c65227d2c7b226e616d65223a227072696d65222c2274797065223a226c6f6e67222c2264656661756c74223a343239333931383732317d2c7b226e616d65223a226e756d6265725f6f665f73657276657273222c2274797065223a22696e74222c2264656661756c74223a327d2c7b226e616d65223a2268616d6d696e675f776569676874222c2274797065223a5b22696e74222c226e756c6c225d7d2c7b226e616d65223a2262617463685f73746172745f74696d65222c2274797065223a7b2274797065223a226c6f6e67222c226c6f676963616c54797065223a2274696d657374616d702d6d696c6c6973227d7d2c7b226e616d65223a2262617463685f656e645f74696d65222c2274797065223a7b2274797065223a226c6f6e67222c226c6f676963616c54797065223a2274696d657374616d702d6d696c6c6973227d7d2c7b226e616d65223a227061636b65745f66696c655f646967657374222c2274797065223a226279746573227d5d7d0010f2cebbc55708399ccbf2a3eb7673400290014862653464663164362d656638362d346339312d383838302d6137393430303238613739341466616b652d62617463680404560e2db29df93f220402f693f1f0058297f1f005020110f2cebbc55708399ccbf2a3eb767340").unwrap(); + let ingestion_header_1 = IngestionHeader { + batch_uuid: Uuid::parse_str("be4df1d6-ef86-4c91-8880-a7940028a794").unwrap(), name: "fake-batch".to_owned(), bins: 2, epsilon: 1.601, @@ -1125,9 +1236,11 @@ mod tests { batch_start_time: 789456123, batch_end_time: 789456321, packet_file_digest: vec![1u8], - }, - IngestionHeader { - batch_uuid: Uuid::new_v4(), + }; + + let ingestion_header_2_bytes = hex::decode("4f626a0104146176726f2e636f646563086e756c6c166176726f2e736368656d61e8097b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f496e67657374696f6e486561646572222c226669656c6473223a5b7b226e616d65223a2262617463685f75756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d2c7b226e616d65223a226e616d65222c2274797065223a22737472696e67227d2c7b226e616d65223a2262696e73222c2274797065223a22696e74227d2c7b226e616d65223a22657073696c6f6e222c2274797065223a22646f75626c65227d2c7b226e616d65223a227072696d65222c2274797065223a226c6f6e67222c2264656661756c74223a343239333931383732317d2c7b226e616d65223a226e756d6265725f6f665f73657276657273222c2274797065223a22696e74222c2264656661756c74223a327d2c7b226e616d65223a2268616d6d696e675f776569676874222c2274797065223a5b22696e74222c226e756c6c225d7d2c7b226e616d65223a2262617463685f73746172745f74696d65222c2274797065223a7b2274797065223a226c6f6e67222c226c6f676963616c54797065223a2274696d657374616d702d6d696c6c6973227d7d2c7b226e616d65223a2262617463685f656e645f74696d65222c2274797065223a7b2274797065223a226c6f6e67222c226c6f676963616c54797065223a2274696d657374616d702d6d696c6c6973227d7d2c7b226e616d65223a227061636b65745f66696c655f646967657374222c2274797065223a226279746573227d5d7d00e30977c811a59cb3e1c2043f8f93c1a30292014830336239646561312d623534382d343032622d393036312d3031333866376337323864621466616b652d62617463680404560e2db29df93f22040018f693f1f0058297f1f0050202e30977c811a59cb3e1c2043f8f93c1a3").unwrap(); + let ingestion_header_2 = IngestionHeader { + batch_uuid: Uuid::parse_str("03b9dea1-b548-402b-9061-0138f7c728db").unwrap(), name: "fake-batch".to_owned(), bins: 2, epsilon: 1.601, @@ -1137,59 +1250,111 @@ mod tests { batch_start_time: 789456123, batch_end_time: 789456321, packet_file_digest: vec![2u8], - }, - ]; + }; - for header in headers { - let mut record_vec = Vec::new(); + vec![ + (ingestion_header_1, ingestion_header_1_bytes), + (ingestion_header_2, ingestion_header_2_bytes), + ] + }; + } + #[test] + fn read_ingestion_header() { + for (want_header, header_bytes) in &*GOLDEN_INGESTION_HEADERS { + let ingestion_header = IngestionHeader::read(&header_bytes[..]).unwrap(); + assert_eq!(want_header, &ingestion_header); + } + } + + #[test] + fn roundtrip_ingestion_header() { + for (header, _) in &*GOLDEN_INGESTION_HEADERS { + let mut record_vec = Vec::new(); header.write(&mut record_vec).expect("write error"); let header_again = IngestionHeader::read(&record_vec[..]).expect("read error"); - assert_eq!(header_again, *header); + assert_eq!(&header_again, header); } } - #[test] - fn roundtrip_data_share_packet() { - let packets = &[ - IngestionDataSharePacket { - uuid: Uuid::new_v4(), + lazy_static! { + static ref GOLDEN_INGESTION_DATA_SHARE_PACKETS: Vec<(IngestionDataSharePacket, Vec)> = { + let ingestion_data_share_packet_1_bytes = hex::decode("4f626a0104146176726f2e636f646563086e756c6c166176726f2e736368656d6198067b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f4461746153686172655061636b6574222c226669656c6473223a5b7b226e616d65223a2275756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d2c7b226e616d65223a22656e637279707465645f7061796c6f6164222c2274797065223a226279746573227d2c7b226e616d65223a22656e6372797074696f6e5f6b65795f6964222c2274797065223a5b226e756c6c222c22737472696e67225d7d2c7b226e616d65223a22725f706974222c2274797065223a226c6f6e67227d2c7b226e616d65223a2276657273696f6e5f636f6e66696775726174696f6e222c2274797065223a5b226e756c6c222c22737472696e67225d7d2c7b226e616d65223a226465766963655f6e6f6e6365222c2274797065223a5b226e756c6c222c226279746573225d7d5d7d009c5da591513530367b96e748024edd3d0284014839363466373234302d643066662d343234642d383966362d3636366463306534626563630800010203021466616b652d6b65792d31020210636f6e6669672d31009c5da591513530367b96e748024edd3d").unwrap(); + let ingestion_data_share_packet_1 = IngestionDataSharePacket { + uuid: Uuid::parse_str("964f7240-d0ff-424d-89f6-666dc0e4becc").unwrap(), encrypted_payload: vec![0u8, 1u8, 2u8, 3u8], encryption_key_id: Some("fake-key-1".to_owned()), r_pit: 1, version_configuration: Some("config-1".to_owned()), device_nonce: None, - }, - IngestionDataSharePacket { - uuid: Uuid::new_v4(), + }; + + let ingestion_data_share_packet_2_bytes = hex::decode("4f626a0104146176726f2e636f646563086e756c6c166176726f2e736368656d6198067b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f4461746153686172655061636b6574222c226669656c6473223a5b7b226e616d65223a2275756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d2c7b226e616d65223a22656e637279707465645f7061796c6f6164222c2274797065223a226279746573227d2c7b226e616d65223a22656e6372797074696f6e5f6b65795f6964222c2274797065223a5b226e756c6c222c22737472696e67225d7d2c7b226e616d65223a22725f706974222c2274797065223a226c6f6e67227d2c7b226e616d65223a2276657273696f6e5f636f6e66696775726174696f6e222c2274797065223a5b226e756c6c222c22737472696e67225d7d2c7b226e616d65223a226465766963655f6e6f6e6365222c2274797065223a5b226e756c6c222c226279746573225d7d5d7d005f3567685778ee120d743d690daaaa6502664830323034623036342d353865382d346666342d613635392d3164326466636334333232340804050607000400020808090a0b5f3567685778ee120d743d690daaaa65").unwrap(); + let ingestion_data_share_packet_2 = IngestionDataSharePacket { + uuid: Uuid::parse_str("0204b064-58e8-4ff4-a659-1d2dfcc43224").unwrap(), encrypted_payload: vec![4u8, 5u8, 6u8, 7u8], encryption_key_id: None, r_pit: 2, version_configuration: None, device_nonce: Some(vec![8u8, 9u8, 10u8, 11u8]), - }, - IngestionDataSharePacket { - uuid: Uuid::new_v4(), + }; + + let ingestion_data_share_packet_3_bytes = hex::decode("4f626a0104166176726f2e736368656d6198067b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f4461746153686172655061636b6574222c226669656c6473223a5b7b226e616d65223a2275756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d2c7b226e616d65223a22656e637279707465645f7061796c6f6164222c2274797065223a226279746573227d2c7b226e616d65223a22656e6372797074696f6e5f6b65795f6964222c2274797065223a5b226e756c6c222c22737472696e67225d7d2c7b226e616d65223a22725f706974222c2274797065223a226c6f6e67227d2c7b226e616d65223a2276657273696f6e5f636f6e66696775726174696f6e222c2274797065223a5b226e756c6c222c22737472696e67225d7d2c7b226e616d65223a226465766963655f6e6f6e6365222c2274797065223a5b226e756c6c222c226279746573225d7d5d7d146176726f2e636f646563086e756c6c00ad4c03ddfc76ee72ebd1d1b9592a5d3a02724864383165646431322d376335302d343939362d613733622d3539303064333734303836620808090a0b021466616b652d6b65792d33060000ad4c03ddfc76ee72ebd1d1b9592a5d3a").unwrap(); + let ingestion_data_share_packet_3 = IngestionDataSharePacket { + uuid: Uuid::parse_str("d81edd12-7c50-4996-a73b-5900d374086b").unwrap(), encrypted_payload: vec![8u8, 9u8, 10u8, 11u8], encryption_key_id: Some("fake-key-3".to_owned()), r_pit: 3, version_configuration: None, device_nonce: None, - }, - ]; + }; + + vec![ + ( + ingestion_data_share_packet_1, + ingestion_data_share_packet_1_bytes, + ), + ( + ingestion_data_share_packet_2, + ingestion_data_share_packet_2_bytes, + ), + ( + ingestion_data_share_packet_3, + ingestion_data_share_packet_3_bytes, + ), + ] + }; + } + + #[test] + fn read_data_share_packet() { + for (want_data_share_packet, data_share_packet_bytes) in + &*GOLDEN_INGESTION_DATA_SHARE_PACKETS + { + let mut reader = Reader::with_schema( + IngestionDataSharePacket::schema(), + &data_share_packet_bytes[..], + ) + .unwrap(); + let data_share_packet = IngestionDataSharePacket::read(&mut reader).unwrap(); + assert_eq!(want_data_share_packet, &data_share_packet); + } + } + #[test] + fn roundtrip_data_share_packet() { let mut record_vec = Vec::new(); - let schema = IngestionDataSharePacket::schema(); - let mut writer = Writer::new(&schema, &mut record_vec); + let mut writer = Writer::new(IngestionDataSharePacket::schema(), &mut record_vec); - for packet in packets { + for (packet, _) in &*GOLDEN_INGESTION_DATA_SHARE_PACKETS { packet.write(&mut writer).expect("write error"); } writer.flush().unwrap(); - let mut reader = Reader::with_schema(&schema, &record_vec[..]).unwrap(); - for packet in packets { + let mut reader = + Reader::with_schema(IngestionDataSharePacket::schema(), &record_vec[..]).unwrap(); + for (packet, _) in &*GOLDEN_INGESTION_DATA_SHARE_PACKETS { let packet_again = IngestionDataSharePacket::read(&mut reader).expect("read error"); assert_eq!(packet_again, *packet); } @@ -1201,11 +1366,11 @@ mod tests { ); } - #[test] - fn roundtrip_validation_header() { - let headers = &[ - ValidationHeader { - batch_uuid: Uuid::new_v4(), + lazy_static! { + static ref GOLDEN_VALIDATION_HEADERS: Vec<(ValidationHeader, Vec)> = { + let validation_header_1_bytes = hex::decode("4f626a0104166176726f2e736368656d619a077b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f56616c6964697479486561646572222c226669656c6473223a5b7b226e616d65223a2262617463685f75756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d2c7b226e616d65223a226e616d65222c2274797065223a22737472696e67227d2c7b226e616d65223a2262696e73222c2274797065223a22696e74227d2c7b226e616d65223a22657073696c6f6e222c2274797065223a22646f75626c65227d2c7b226e616d65223a227072696d65222c2274797065223a226c6f6e67222c2264656661756c74223a343239333931383732317d2c7b226e616d65223a226e756d6265725f6f665f73657276657273222c2274797065223a22696e74222c2264656661756c74223a327d2c7b226e616d65223a2268616d6d696e675f776569676874222c2274797065223a5b22696e74222c226e756c6c225d7d2c7b226e616d65223a227061636b65745f66696c655f646967657374222c2274797065223a226279746573227d5d7d146176726f2e636f646563086e756c6c009f8fc651b8e2191bdfa8ad453ea2b73c027c4838663566306637392d303838372d343332322d386238632d3634646165623765353138331466616b652d62617463680404560e2db29df93f22040202049f8fc651b8e2191bdfa8ad453ea2b73c").unwrap(); + let validation_header_1 = ValidationHeader { + batch_uuid: Uuid::parse_str("8f5f0f79-0887-4322-8b8c-64daeb7e5183").unwrap(), name: "fake-batch".to_owned(), bins: 2, epsilon: 1.601, @@ -1213,9 +1378,11 @@ mod tests { number_of_servers: 2, hamming_weight: None, packet_file_digest: vec![4u8], - }, - ValidationHeader { - batch_uuid: Uuid::new_v4(), + }; + + let validation_header_2_bytes = hex::decode("4f626a0104166176726f2e736368656d619a077b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f56616c6964697479486561646572222c226669656c6473223a5b7b226e616d65223a2262617463685f75756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d2c7b226e616d65223a226e616d65222c2274797065223a22737472696e67227d2c7b226e616d65223a2262696e73222c2274797065223a22696e74227d2c7b226e616d65223a22657073696c6f6e222c2274797065223a22646f75626c65227d2c7b226e616d65223a227072696d65222c2274797065223a226c6f6e67222c2264656661756c74223a343239333931383732317d2c7b226e616d65223a226e756d6265725f6f665f73657276657273222c2274797065223a22696e74222c2264656661756c74223a327d2c7b226e616d65223a2268616d6d696e675f776569676874222c2274797065223a5b22696e74222c226e756c6c225d7d2c7b226e616d65223a227061636b65745f66696c655f646967657374222c2274797065223a226279746573227d5d7d146176726f2e636f646563086e756c6c00d45f391c5edf41c4b15025c8a6645177027e4834636464623330322d343462322d343565392d613164312d6438383464663862383765301466616b652d62617463680404560e2db29df93f220400180206d45f391c5edf41c4b15025c8a6645177").unwrap(); + let validation_header_2 = ValidationHeader { + batch_uuid: Uuid::parse_str("4cddb302-44b2-45e9-a1d1-d884df8b87e0").unwrap(), name: "fake-batch".to_owned(), bins: 2, epsilon: 1.601, @@ -1223,53 +1390,91 @@ mod tests { number_of_servers: 2, hamming_weight: Some(12), packet_file_digest: vec![6u8], - }, - ]; + }; - for header in headers { - let mut record_vec = Vec::new(); + vec![ + (validation_header_1, validation_header_1_bytes), + (validation_header_2, validation_header_2_bytes), + ] + }; + } + #[test] + fn read_validation_header() { + for (want_header, header_bytes) in &*GOLDEN_VALIDATION_HEADERS { + let validation_header = ValidationHeader::read(&header_bytes[..]).unwrap(); + assert_eq!(want_header, &validation_header); + } + } + + #[test] + fn roundtrip_validation_header() { + for (header, _) in &*GOLDEN_VALIDATION_HEADERS { + let mut record_vec = Vec::new(); header.write(&mut record_vec).expect("write error"); let header_again = ValidationHeader::read(&record_vec[..]).expect("read error"); assert_eq!(header_again, *header); } } - #[test] - fn roundtrip_validation_packet() { - let packets = &[ - ValidationPacket { - uuid: Uuid::new_v4(), + lazy_static! { + static ref GOLDEN_VALIDATION_PACKETS: Vec<(ValidationPacket, Vec)> = { + let validation_packet_1_bytes = hex::decode("4f626a0104166176726f2e736368656d61ee037b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f56616c69646974795061636b6574222c226669656c6473223a5b7b226e616d65223a2275756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d2c7b226e616d65223a22665f72222c2274797065223a226c6f6e67227d2c7b226e616d65223a22675f72222c2274797065223a226c6f6e67227d2c7b226e616d65223a22685f72222c2274797065223a226c6f6e67227d5d7d146176726f2e636f646563086e756c6c00368cb0feb3f809181cea03dcb7178a2502504835316239643938312d393537342d343466622d616533352d383463636631336366306639020406368cb0feb3f809181cea03dcb7178a25").unwrap(); + let validation_packet_1 = ValidationPacket { + uuid: Uuid::parse_str("51b9d981-9574-44fb-ae35-84ccf13cf0f9").unwrap(), f_r: 1, g_r: 2, h_r: 3, - }, - ValidationPacket { - uuid: Uuid::new_v4(), + }; + + let validation_packet_2_bytes = hex::decode("4f626a0104146176726f2e636f646563086e756c6c166176726f2e736368656d61ee037b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f56616c69646974795061636b6574222c226669656c6473223a5b7b226e616d65223a2275756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d2c7b226e616d65223a22665f72222c2274797065223a226c6f6e67227d2c7b226e616d65223a22675f72222c2274797065223a226c6f6e67227d2c7b226e616d65223a22685f72222c2274797065223a226c6f6e67227d5d7d00ce6c0aad9e40580f01836ff18ffc13d302504863306532343832642d656533622d343864342d623333302d356566623464643035323638080a0cce6c0aad9e40580f01836ff18ffc13d3").unwrap(); + let validation_packet_2 = ValidationPacket { + uuid: Uuid::parse_str("c0e2482d-ee3b-48d4-b330-5efb4dd05268").unwrap(), f_r: 4, g_r: 5, h_r: 6, - }, - ValidationPacket { - uuid: Uuid::new_v4(), + }; + + let validation_packet_3_bytes = hex::decode("4f626a0104166176726f2e736368656d61ee037b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f56616c69646974795061636b6574222c226669656c6473223a5b7b226e616d65223a2275756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d2c7b226e616d65223a22665f72222c2274797065223a226c6f6e67227d2c7b226e616d65223a22675f72222c2274797065223a226c6f6e67227d2c7b226e616d65223a22685f72222c2274797065223a226c6f6e67227d5d7d146176726f2e636f646563086e756c6c008f37d54782b5a4ba978ca4b04c41d54102504865656365653134622d613030642d343766622d613031382d3130643562316536303739370e10128f37d54782b5a4ba978ca4b04c41d541").unwrap(); + let validation_packet_3 = ValidationPacket { + uuid: Uuid::parse_str("eecee14b-a00d-47fb-a018-10d5b1e60797").unwrap(), f_r: 7, g_r: 8, h_r: 9, - }, - ]; + }; + + vec![ + (validation_packet_1, validation_packet_1_bytes), + (validation_packet_2, validation_packet_2_bytes), + (validation_packet_3, validation_packet_3_bytes), + ] + }; + } + #[test] + fn read_validation_packet() { + for (want_validation_packet, validation_packet_bytes) in &*GOLDEN_VALIDATION_PACKETS { + let mut reader = + Reader::with_schema(ValidationPacket::schema(), &validation_packet_bytes[..]) + .unwrap(); + let data_share_packet = ValidationPacket::read(&mut reader).unwrap(); + assert_eq!(want_validation_packet, &data_share_packet); + } + } + + #[test] + fn roundtrip_validation_packet() { let mut record_vec = Vec::new(); - let schema = ValidationPacket::schema(); - let mut writer = Writer::new(&schema, &mut record_vec); + let mut writer = Writer::new(ValidationPacket::schema(), &mut record_vec); - for packet in packets { + for (packet, _) in &*GOLDEN_VALIDATION_PACKETS { packet.write(&mut writer).expect("write error"); } writer.flush().unwrap(); - let mut reader = Reader::with_schema(&schema, &record_vec[..]).unwrap(); - for packet in packets { + let mut reader = Reader::with_schema(ValidationPacket::schema(), &record_vec[..]).unwrap(); + for (packet, _) in &*GOLDEN_VALIDATION_PACKETS { let packet_again = ValidationPacket::read(&mut reader).expect("read error"); assert_eq!(packet_again, *packet); } @@ -1278,11 +1483,14 @@ mod tests { assert_matches!(ValidationPacket::read(&mut reader), Err(Error::EofError)); } - #[test] - fn roundtrip_sum_part() { - let headers = &[ - SumPart { - batch_uuids: vec![Uuid::new_v4(), Uuid::new_v4()], + lazy_static! { + static ref GOLDEN_SUM_PARTS: Vec<(SumPart, Vec)> = { + let sum_part_1_bytes = hex::decode("4f626a0104166176726f2e736368656d61b20b7b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f53756d50617274222c226669656c6473223a5b7b226e616d65223a2262617463685f7575696473222c2274797065223a7b2274797065223a226172726179222c226974656d73223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d7d2c7b226e616d65223a226e616d65222c2274797065223a22737472696e67227d2c7b226e616d65223a2262696e73222c2274797065223a22696e74227d2c7b226e616d65223a22657073696c6f6e222c2274797065223a22646f75626c65227d2c7b226e616d65223a227072696d65222c2274797065223a226c6f6e67227d2c7b226e616d65223a226e756d6265725f6f665f73657276657273222c2274797065223a22696e74227d2c7b226e616d65223a2268616d6d696e675f776569676874222c2274797065223a5b22696e74222c226e756c6c225d7d2c7b226e616d65223a2273756d222c2274797065223a7b2274797065223a226172726179222c226974656d73223a226c6f6e67227d7d2c7b226e616d65223a226167677265676174696f6e5f73746172745f74696d65222c2274797065223a7b2274797065223a226c6f6e67222c226c6f676963616c54797065223a2274696d657374616d702d6d696c6c6973227d7d2c7b226e616d65223a226167677265676174696f6e5f656e645f74696d65222c2274797065223a7b2274797065223a226c6f6e67222c226c6f676963616c54797065223a2274696d657374616d702d6d696c6c6973227d7d2c7b226e616d65223a227061636b65745f66696c655f646967657374222c2274797065223a226279746573227d2c7b226e616d65223a22746f74616c5f696e646976696475616c5f636c69656e7473222c2274797065223a226c6f6e67227d5d7d146176726f2e636f646563086e756c6c00ae60e74a8b6b11559ed0c9e7de221c0102ee01044864623562316135632d623563362d346439332d623736392d3766353765366361663538644830363265373931622d396132312d343466332d383561352d656335323333643636383662001466616b652d62617463680404560e2db29df93f22040206181a1c00f693f1f0058297f1f0050601020304ae60e74a8b6b11559ed0c9e7de221c01").unwrap(); + let sum_part_1 = SumPart { + batch_uuids: vec![ + Uuid::parse_str("db5b1a5c-b5c6-4d93-b769-7f57e6caf58d").unwrap(), + Uuid::parse_str("062e791b-9a21-44f3-85a5-ec5233d6686b").unwrap(), + ], name: "fake-batch".to_owned(), bins: 2, epsilon: 1.601, @@ -1294,9 +1502,11 @@ mod tests { aggregation_end_time: 789456321, packet_file_digest: vec![1, 2, 3], total_individual_clients: 2, - }, - SumPart { - batch_uuids: vec![Uuid::new_v4()], + }; + + let sum_part_2_bytes = hex::decode("4f626a0104166176726f2e736368656d61b20b7b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f53756d50617274222c226669656c6473223a5b7b226e616d65223a2262617463685f7575696473222c2274797065223a7b2274797065223a226172726179222c226974656d73223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d7d2c7b226e616d65223a226e616d65222c2274797065223a22737472696e67227d2c7b226e616d65223a2262696e73222c2274797065223a22696e74227d2c7b226e616d65223a22657073696c6f6e222c2274797065223a22646f75626c65227d2c7b226e616d65223a227072696d65222c2274797065223a226c6f6e67227d2c7b226e616d65223a226e756d6265725f6f665f73657276657273222c2274797065223a22696e74227d2c7b226e616d65223a2268616d6d696e675f776569676874222c2274797065223a5b22696e74222c226e756c6c225d7d2c7b226e616d65223a2273756d222c2274797065223a7b2274797065223a226172726179222c226974656d73223a226c6f6e67227d7d2c7b226e616d65223a226167677265676174696f6e5f73746172745f74696d65222c2274797065223a7b2274797065223a226c6f6e67222c226c6f676963616c54797065223a2274696d657374616d702d6d696c6c6973227d7d2c7b226e616d65223a226167677265676174696f6e5f656e645f74696d65222c2274797065223a7b2274797065223a226c6f6e67222c226c6f676963616c54797065223a2274696d657374616d702d6d696c6c6973227d7d2c7b226e616d65223a227061636b65745f66696c655f646967657374222c2274797065223a226279746573227d2c7b226e616d65223a22746f74616c5f696e646976696475616c5f636c69656e7473222c2274797065223a226c6f6e67227d5d7d146176726f2e636f646563086e756c6c0043a570b58e473dfef2dfd26b0bff5e7902a601024835643662343237612d393932372d343137662d383035622d373765373666323663343738001466616b652d62617463680404560e2db29df93f2204001806181a1c00f693f1f0058297f1f005060708090443a570b58e473dfef2dfd26b0bff5e79").unwrap(); + let sum_part_2 = SumPart { + batch_uuids: vec![Uuid::parse_str("5d6b427a-9927-417f-805b-77e76f26c478").unwrap()], name: "fake-batch".to_owned(), bins: 2, epsilon: 1.601, @@ -1308,44 +1518,81 @@ mod tests { aggregation_end_time: 789456321, packet_file_digest: vec![7, 8, 9], total_individual_clients: 2, - }, - ]; + }; + + vec![ + (sum_part_1, sum_part_1_bytes), + (sum_part_2, sum_part_2_bytes), + ] + }; + } + + #[test] + fn read_sum_part() { + for (want_sum_part, sum_part_bytes) in &*GOLDEN_SUM_PARTS { + let sum_part = SumPart::read(&sum_part_bytes[..]).unwrap(); + assert_eq!(want_sum_part, &sum_part); + } + } - for header in headers { + #[test] + fn roundtrip_sum_part() { + for (sum_part, _) in &*GOLDEN_SUM_PARTS { let mut record_vec = Vec::new(); + sum_part.write(&mut record_vec).expect("write error"); + let sum_part_again = SumPart::read(&record_vec[..]).expect("read error"); + assert_eq!(sum_part_again, *sum_part); + } + } - header.write(&mut record_vec).expect("write error"); - let header_again = SumPart::read(&record_vec[..]).expect("read error"); - assert_eq!(header_again, *header); + lazy_static! { + static ref GOLDEN_INVALID_PACKETS: Vec<(InvalidPacket, Vec)> = { + let invalid_packet_1_bytes = hex::decode("4f626a0104146176726f2e636f646563086e756c6c166176726f2e736368656d61be027b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f496e76616c69645061636b6574222c226669656c6473223a5b7b226e616d65223a2275756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d5d7d006de170c446ec2172338211e7aa0ad82f024a4861623763326435622d353633342d343730662d383162352d3063656164303639616162346de170c446ec2172338211e7aa0ad82f").unwrap(); + let invalid_packet_1 = InvalidPacket { + uuid: Uuid::parse_str("ab7c2d5b-5634-470f-81b5-0cead069aab4").unwrap(), + }; + + let invalid_packet_2_bytes = hex::decode("4f626a0104166176726f2e736368656d61be027b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f496e76616c69645061636b6574222c226669656c6473223a5b7b226e616d65223a2275756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d5d7d146176726f2e636f646563086e756c6c0095969d036d8e3d0f39a1393b221e6aae024a4839356265386139302d313232642d343663632d396361372d66366535663934316161376295969d036d8e3d0f39a1393b221e6aae").unwrap(); + let invalid_packet_2 = InvalidPacket { + uuid: Uuid::parse_str("95be8a90-122d-46cc-9ca7-f6e5f941aa7b").unwrap(), + }; + + let invalid_packet_3_bytes = hex::decode("4f626a0104166176726f2e736368656d61be027b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f496e76616c69645061636b6574222c226669656c6473223a5b7b226e616d65223a2275756964222c2274797065223a7b2274797065223a22737472696e67222c226c6f676963616c54797065223a2275756964227d7d5d7d146176726f2e636f646563086e756c6c00d8f3eec9d4c68061ec330d794ea704ad024a4837366162353839362d636132352d343432632d396338302d633463663766396164393835d8f3eec9d4c68061ec330d794ea704ad").unwrap(); + let invalid_packet_3 = InvalidPacket { + uuid: Uuid::parse_str("76ab5896-ca25-442c-9c80-c4cf7f9ad985").unwrap(), + }; + + vec![ + (invalid_packet_1, invalid_packet_1_bytes), + (invalid_packet_2, invalid_packet_2_bytes), + (invalid_packet_3, invalid_packet_3_bytes), + ] + }; + } + + #[test] + fn read_invalid_packet() { + for (want_invalid_packet, invalid_packet_bytes) in &*GOLDEN_INVALID_PACKETS { + let mut reader = + Reader::with_schema(InvalidPacket::schema(), &invalid_packet_bytes[..]).unwrap(); + let invalid_packet = InvalidPacket::read(&mut reader).unwrap(); + assert_eq!(want_invalid_packet, &invalid_packet); } } #[test] fn roundtrip_invalid_packet() { - let packets = &[ - InvalidPacket { - uuid: Uuid::new_v4(), - }, - InvalidPacket { - uuid: Uuid::new_v4(), - }, - InvalidPacket { - uuid: Uuid::new_v4(), - }, - ]; - let mut record_vec = Vec::new(); - let schema = InvalidPacket::schema(); - let mut writer = Writer::new(&schema, &mut record_vec); + let mut writer = Writer::new(InvalidPacket::schema(), &mut record_vec); - for packet in packets { + for (packet, _) in &*GOLDEN_INVALID_PACKETS { packet.write(&mut writer).expect("write error"); } writer.flush().unwrap(); - let mut reader = Reader::with_schema(&schema, &record_vec[..]).unwrap(); - for packet in packets { + let mut reader = Reader::with_schema(InvalidPacket::schema(), &record_vec[..]).unwrap(); + for (packet, _) in &*GOLDEN_INVALID_PACKETS { let packet_again = InvalidPacket::read(&mut reader).expect("read error"); assert_eq!(packet_again, *packet); } @@ -1353,4 +1600,51 @@ mod tests { // Do one more read. This should yield EOF. assert_matches!(InvalidPacket::read(&mut reader), Err(Error::EofError)); } + + lazy_static! { + static ref GOLDEN_VALIDATION_BATCHES: Vec<(ValidationBatch, Vec)> = { + let golden_validation_batch_1_bytes = hex::decode("4f626a0104166176726f2e736368656d61f0057b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f56616c69646974794261746368222c226669656c6473223a5b7b226e616d65223a22736967222c2274797065223a7b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f42617463685369676e6174757265222c226669656c6473223a5b7b226e616d65223a2262617463685f6865616465725f7369676e6174757265222c2274797065223a226279746573227d2c7b226e616d65223a226b65795f6964656e746966696572222c2274797065223a22737472696e67227d5d7d7d2c7b226e616d65223a22686561646572222c2274797065223a226279746573227d2c7b226e616d65223a227061636b657473222c2274797065223a226279746573227d5d7d146176726f2e636f646563086e756c6c005d68da833f0afa80e6cc62c07de60696023006010203166d792d636f6f6c2d6b657906040506060708095d68da833f0afa80e6cc62c07de60696").unwrap(); + let golden_validation_batch_1 = ValidationBatch { + sig: BatchSignature { + batch_header_signature: vec![1, 2, 3], + key_identifier: "my-cool-key".to_owned(), + }, + header: vec![4, 5, 6], + packets: vec![7, 8, 9], + }; + + let golden_validation_batch_2_bytes = hex::decode("4f626a0104146176726f2e636f646563086e756c6c166176726f2e736368656d61f0057b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f56616c69646974794261746368222c226669656c6473223a5b7b226e616d65223a22736967222c2274797065223a7b2274797065223a227265636f7264222c226e616d657370616365223a226f72672e61626574746572696e7465726e65742e7072696f2e7631222c226e616d65223a225072696f42617463685369676e6174757265222c226669656c6473223a5b7b226e616d65223a2262617463685f6865616465725f7369676e6174757265222c2274797065223a226279746573227d2c7b226e616d65223a226b65795f6964656e746966696572222c2274797065223a22737472696e67227d5d7d7d2c7b226e616d65223a22686561646572222c2274797065223a226279746573227d2c7b226e616d65223a227061636b657473222c2274797065223a226279746573227d5d7d006cc69b10d7b8900c362cd1f49e6b125a0230060a0b0c166d792d636f6f6c2d6b6579060d0e0f061011126cc69b10d7b8900c362cd1f49e6b125a").unwrap(); + let golden_validation_batch_2 = ValidationBatch { + sig: BatchSignature { + batch_header_signature: vec![10, 11, 12], + key_identifier: "my-cool-key".to_owned(), + }, + header: vec![13, 14, 15], + packets: vec![16, 17, 18], + }; + + vec![ + (golden_validation_batch_1, golden_validation_batch_1_bytes), + (golden_validation_batch_2, golden_validation_batch_2_bytes), + ] + }; + } + + #[test] + fn read_validation_batch() { + for (want_batch, batch_bytes) in &*GOLDEN_VALIDATION_BATCHES { + let validation_batch = ValidationBatch::read(&batch_bytes[..]).unwrap(); + assert_eq!(want_batch, &validation_batch); + } + } + + #[test] + fn roundtrip_validation_batch() { + for (want_batch, _) in &*GOLDEN_VALIDATION_BATCHES { + let mut record_vec = Vec::new(); + want_batch.write(&mut record_vec).expect("write error"); + let got_batch = ValidationBatch::read(&record_vec[..]).expect("read error"); + assert_eq!(want_batch, &got_batch); + } + } } diff --git a/facilitator/src/lib.rs b/facilitator/src/lib.rs index 92b188bae..a58e0e1ca 100644 --- a/facilitator/src/lib.rs +++ b/facilitator/src/lib.rs @@ -33,6 +33,8 @@ pub enum Error { MalformedHeaderError(String), #[error("malformed data packet: {0}")] MalformedDataPacketError(String), + #[error("malformed batch: {0}")] + MalformedBatchError(String), #[error("end of file")] EofError, #[error("HTTP resource error")]