From d13d9517cd964f10ae3436cb0437b735f9a41f5c Mon Sep 17 00:00:00 2001 From: Patrick Beza Date: Mon, 16 Sep 2024 11:55:01 +0200 Subject: [PATCH] feat(tee): use hex serialization for RPC responses Following Anton's suggestion, we have switched to hex serialization for API/RPC requests and responses. Previously, we used default JSON serialization for Vec, which resulted in a lengthy comma-separated list of integers. This change will make serialization more efficient and reduce the size of the responses. Then: ``` curl -X POST\ -H "Content-Type: application/json" \ --data '{"jsonrpc": "2.0", "id": 1, "method": "unstable_getTeeProofs", "params": [491882, "Sgx"] }' \ https://mainnet.era.zksync.io {"jsonrpc":"2.0","result":[{"attestation":[3,0,2,0,0,0,0,0,10, ``` Now: ``` $ curl -X POST \ -H "Content-Type: application/json" \ --data '{"jsonrpc": "2.0", "id": 1, "method": "unstable_getTeeProofs", "params": [1, "sgx"] }' \ http://localhost:3050 {"jsonrpc":"2.0","result":[{"l1BatchNumber":1,"teeType":"sgx","pubkey":"0506070809","signature":"0001020304","proof":"0a0b0c0d0e","provedAt":"2024-09-16T11:53:38.253033Z","attestation":"0403020100"}],"id":1} ``` --- Cargo.lock | 1 + core/lib/prover_interface/Cargo.toml | 2 +- core/lib/prover_interface/src/api.rs | 4 ++ core/lib/prover_interface/src/outputs.rs | 5 ++ .../tests/job_serialization.rs | 6 +- core/lib/types/Cargo.toml | 1 + core/lib/types/src/api/mod.rs | 6 ++ core/node/api_server/src/web3/testonly.rs | 2 +- core/node/api_server/src/web3/tests/mod.rs | 1 + .../api_server/src/web3/tests/unstable.rs | 70 +++++++++++++++++++ core/node/proof_data_handler/src/tests.rs | 6 +- 11 files changed, 96 insertions(+), 8 deletions(-) create mode 100644 core/node/api_server/src/web3/tests/unstable.rs diff --git a/Cargo.lock b/Cargo.lock index 2aaf875a2f49..90849bc72b37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10898,6 +10898,7 @@ dependencies = [ "secp256k1", "serde", "serde_json", + "serde_with", "strum", "thiserror", "tokio", diff --git a/core/lib/prover_interface/Cargo.toml b/core/lib/prover_interface/Cargo.toml index 8c73c2c6ac38..889b80b4fbee 100644 --- a/core/lib/prover_interface/Cargo.toml +++ b/core/lib/prover_interface/Cargo.toml @@ -20,7 +20,7 @@ circuit_sequencer_api_1_5_0.workspace = true serde.workspace = true strum = { workspace = true, features = ["derive"] } -serde_with = { workspace = true, features = ["base64"] } +serde_with = { workspace = true, features = ["base64", "hex"] } chrono = { workspace = true, features = ["serde"] } [dev-dependencies] diff --git a/core/lib/prover_interface/src/api.rs b/core/lib/prover_interface/src/api.rs index bc95345bbbaa..776cd3141cbe 100644 --- a/core/lib/prover_interface/src/api.rs +++ b/core/lib/prover_interface/src/api.rs @@ -2,6 +2,7 @@ //! This module defines the types used in the API. use serde::{Deserialize, Serialize}; +use serde_with::{hex::Hex, serde_as}; use zksync_types::{ protocol_version::{L1VerifierConfig, ProtocolSemanticVersion}, tee_types::TeeType, @@ -71,8 +72,11 @@ pub struct VerifyProofRequest(pub Box); #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct SubmitTeeProofRequest(pub Box); +#[serde_as] #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct RegisterTeeAttestationRequest { + #[serde_as(as = "Hex")] pub attestation: Vec, + #[serde_as(as = "Hex")] pub pubkey: Vec, } diff --git a/core/lib/prover_interface/src/outputs.rs b/core/lib/prover_interface/src/outputs.rs index 9672bfb2142b..60a9eaba760b 100644 --- a/core/lib/prover_interface/src/outputs.rs +++ b/core/lib/prover_interface/src/outputs.rs @@ -2,6 +2,7 @@ use core::fmt; use circuit_sequencer_api_1_5_0::proof::FinalProof; use serde::{Deserialize, Serialize}; +use serde_with::{hex::Hex, serde_as}; use zksync_object_store::{serialize_using_bincode, Bucket, StoredObject}; use zksync_types::{protocol_version::ProtocolSemanticVersion, tee_types::TeeType, L1BatchNumber}; @@ -14,14 +15,18 @@ pub struct L1BatchProofForL1 { } /// A "final" TEE proof that can be sent to the L1 contract. +#[serde_as] #[derive(Clone, PartialEq, Serialize, Deserialize)] pub struct L1BatchTeeProofForL1 { // signature generated within the TEE enclave, using the privkey corresponding to the pubkey + #[serde_as(as = "Hex")] pub signature: Vec, // pubkey used for signature verification; each key pair is attested by the TEE attestation // stored in the db + #[serde_as(as = "Hex")] pub pubkey: Vec, // data that was signed + #[serde_as(as = "Hex")] pub proof: Vec, // type of TEE used for attestation pub tee_type: TeeType, diff --git a/core/lib/prover_interface/tests/job_serialization.rs b/core/lib/prover_interface/tests/job_serialization.rs index a2aee0c2733e..ead59749abe3 100644 --- a/core/lib/prover_interface/tests/job_serialization.rs +++ b/core/lib/prover_interface/tests/job_serialization.rs @@ -167,9 +167,9 @@ fn test_proof_request_serialization() { #[test] fn test_tee_proof_request_serialization() { let tee_proof_str = r#"{ - "signature": [ 0, 1, 2, 3, 4 ], - "pubkey": [ 5, 6, 7, 8, 9 ], - "proof": [ 10, 11, 12, 13, 14 ], + "signature": "0001020304", + "pubkey": "0506070809", + "proof": "0A0B0C0D0E", "tee_type": "sgx" }"#; let tee_proof_result = serde_json::from_str::(tee_proof_str).unwrap(); diff --git a/core/lib/types/Cargo.toml b/core/lib/types/Cargo.toml index 55cbef761ad5..54c38384a7ad 100644 --- a/core/lib/types/Cargo.toml +++ b/core/lib/types/Cargo.toml @@ -28,6 +28,7 @@ once_cell.workspace = true rlp.workspace = true serde.workspace = true serde_json.workspace = true +serde_with = { workspace = true, features = ["hex"] } bigdecimal.workspace = true strum = { workspace = true, features = ["derive"] } thiserror.workspace = true diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index 916fae6a35bc..f648204ca557 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -1,6 +1,7 @@ use chrono::{DateTime, Utc}; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; +use serde_with::{hex::Hex, serde_as}; use strum::Display; use zksync_basic_types::{ tee_types::TeeType, @@ -784,15 +785,20 @@ pub struct Proof { pub storage_proof: Vec, } +#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TeeProof { pub l1_batch_number: L1BatchNumber, pub tee_type: Option, + #[serde_as(as = "Option")] pub pubkey: Option>, + #[serde_as(as = "Option")] pub signature: Option>, + #[serde_as(as = "Option")] pub proof: Option>, pub proved_at: DateTime, + #[serde_as(as = "Option")] pub attestation: Option>, } diff --git a/core/node/api_server/src/web3/testonly.rs b/core/node/api_server/src/web3/testonly.rs index a77498d4341d..18ee3a641d0a 100644 --- a/core/node/api_server/src/web3/testonly.rs +++ b/core/node/api_server/src/web3/testonly.rs @@ -182,7 +182,7 @@ async fn spawn_server( let (pub_sub_events_sender, pub_sub_events_receiver) = mpsc::unbounded_channel(); let mut namespaces = Namespace::DEFAULT.to_vec(); - namespaces.extend([Namespace::Debug, Namespace::Snapshots]); + namespaces.extend([Namespace::Debug, Namespace::Snapshots, Namespace::Unstable]); let server_builder = match transport { ApiTransportLabel::Http => ApiBuilder::jsonrpsee_backend(api_config, pool).http(0), diff --git a/core/node/api_server/src/web3/tests/mod.rs b/core/node/api_server/src/web3/tests/mod.rs index 635620e9c525..fe90f1483a5a 100644 --- a/core/node/api_server/src/web3/tests/mod.rs +++ b/core/node/api_server/src/web3/tests/mod.rs @@ -63,6 +63,7 @@ use crate::web3::testonly::{spawn_http_server, spawn_ws_server}; mod debug; mod filters; mod snapshots; +mod unstable; mod vm; mod ws; diff --git a/core/node/api_server/src/web3/tests/unstable.rs b/core/node/api_server/src/web3/tests/unstable.rs new file mode 100644 index 000000000000..e999ac3904d4 --- /dev/null +++ b/core/node/api_server/src/web3/tests/unstable.rs @@ -0,0 +1,70 @@ +//! Tests for the `unstable` Web3 namespace. + +use zksync_types::tee_types::TeeType; +use zksync_web3_decl::namespaces::UnstableNamespaceClient; + +use super::*; + +#[derive(Debug)] +struct GetTeeProofsTest {} + +impl GetTeeProofsTest { + fn new() -> Self { + Self {} + } +} + +#[async_trait] +impl HttpTest for GetTeeProofsTest { + async fn test( + &self, + client: &DynClient, + pool: &ConnectionPool, + ) -> anyhow::Result<()> { + let batch_no = L1BatchNumber(1337); + let tee_type = TeeType::Sgx; + let proof = client.tee_proofs(batch_no, Some(tee_type)).await?; + + assert!(proof.is_empty()); + + let mut storage = pool.connection().await.unwrap(); + storage + .tee_verifier_input_producer_dal() + .create_tee_verifier_input_producer_job(batch_no) + .await?; + + let pubkey = vec![0xDE, 0xAD, 0xBE, 0xEF]; + let attestation = vec![0xC0, 0xFF, 0xEE]; + let mut tee_proof_generation_dal = storage.tee_proof_generation_dal(); + tee_proof_generation_dal + .save_attestation(&pubkey, &attestation) + .await?; + tee_proof_generation_dal + .insert_tee_proof_generation_job(batch_no, tee_type) + .await?; + + let signature = vec![0, 1, 2, 3, 4]; + let pubkey = vec![5, 6, 7, 8, 9]; + let proof_vec = vec![10, 11, 12, 13, 14]; + tee_proof_generation_dal + .save_proof_artifacts_metadata(batch_no, tee_type, &pubkey, &signature, &proof_vec) + .await?; + + let proofs = client.tee_proofs(batch_no, Some(tee_type)).await?; + assert!(proofs.len() == 1); + let proof = &proofs[0]; + assert!(proof.l1_batch_number == batch_no); + assert!(proof.tee_type == Some(tee_type)); + assert!(proof.pubkey.as_ref() == Some(&pubkey)); + assert!(proof.signature.as_ref() == Some(&signature)); + assert!(proof.proof.as_ref() == Some(&proof_vec)); + assert!(proof.attestation.is_none()); + + Ok(()) + } +} + +#[tokio::test] +async fn get_tee_proofs() { + test_http_server(GetTeeProofsTest::new()).await; +} diff --git a/core/node/proof_data_handler/src/tests.rs b/core/node/proof_data_handler/src/tests.rs index 6ab7e4dec436..86cc53234486 100644 --- a/core/node/proof_data_handler/src/tests.rs +++ b/core/node/proof_data_handler/src/tests.rs @@ -131,9 +131,9 @@ async fn submit_tee_proof() { // send a request to the /tee/submit_proofs endpoint, using a mocked TEE proof let tee_proof_request_str = r#"{ - "signature": [ 0, 1, 2, 3, 4 ], - "pubkey": [ 5, 6, 7, 8, 9 ], - "proof": [ 10, 11, 12, 13, 14 ], + "signature": "0001020304", + "pubkey": "0506070809", + "proof": "0A0B0C0D0E", "tee_type": "sgx" }"#; let tee_proof_request =