refactor: Refactor public parameters handling and caching

This adapts to upstream lurk-lang/arecibo#285, which removes the need for separate `ProverKey` and `VerifierKey` instances (the former containing `CommitmentKey` copies) in Public Parameters.

- Implemented structural changes in `nova.rs` for improvements in key handling for the prover and verifier keys, which generated as `OnceCell`,

- Revamped `disk_cache.rs` with added `Arc` imports, elimination of lifetime parameter `'a` from `DiskCache`, modifications of `read` and `write` functions, and updates in the serialization and deserialization steps.
- Altered `public_params` and `supernova_public_params` functions in `mod.rs` to deal more capably with errors and modified various methods and tests to align with the changes in `disk_cache.rs`.
- Removed complete `mem_cache.rs` file, which was unsafe, and unused.

- Moved initialization of `store` to an earlier point in `fibonacci_prove` function and made minor formatting changes in `fibonacci.rs`.

Cargo.toml

@@ -125,7 +125,7 @@ clap = "4.3.17"
 ff = "0.13"
 metrics = "0.22.0"
 neptune = { git = "https://github.com/lurk-lab/neptune", branch = "dev", features = ["abomonation"] }
-nova = { git = "https://github.com/lurk-lab/arecibo", branch = "dev", package = "arecibo" }
+nova = { git = "https://github.com/lurk-lab/arecibo", branch = "killing_params", package = "arecibo", features = ["abomonate"] }
 once_cell = "1.18.0"
 pairing = { version = "0.23" }
 pasta_curves = { git = "https://github.com/lurk-lab/pasta_curves", branch = "dev" }

benches/fibonacci.rs

@@ -90,6 +90,7 @@ fn fibonacci_prove<M: measurement::Measurement>(
         true,
         Kind::NovaPublicParams,
     );
+    let store = Store::default();
     let pp = public_params(&instance).unwrap();
 
     // Track the number of `Lurk frames / sec`
@@ -103,8 +104,6 @@ fn fibonacci_prove<M: measurement::Measurement>(
         BenchmarkId::new(name, params),
         &prove_params,
         |b, prove_params| {
-            let store = Store::default();
-
             let ptr = fib_expr::<pasta_curves::Fq>(&store);
             let prover = NovaProver::new(prove_params.reduction_count, lang_rc.clone());
 
@@ -119,7 +118,7 @@ fn fibonacci_prove<M: measurement::Measurement>(
                     let _ = black_box(result);
                 },
                 BatchSize::LargeInput,
-            )
+            );
         },
     );
 }

benches/sha256.rs

@@ -111,7 +111,7 @@ fn sha256_ivc_prove<M: measurement::Measurement>(
     let lurk_step = make_eval_step_from_config(&EvalConfig::new_ivc(&lang));
 
     // use cached public params
-    let instance: Instance<'_, Fr, Sha256Coproc<Fr>> = Instance::new(
+    let instance: Instance<Fr, Sha256Coproc<Fr>> = Instance::new(
         reduction_count,
         lang_rc.clone(),
         true,

examples/sha256_ivc.rs

@@ -77,7 +77,7 @@ fn main() {
 
     let pp_start = Instant::now();
     let instance = Instance::new(REDUCTION_COUNT, lang_rc, true, Kind::NovaPublicParams);
-    // see the documentation on `with_public_params`
+    // see the documentation on `public_params`
     let pp = public_params(&instance).unwrap();
     let pp_end = pp_start.elapsed();
     println!("Public parameters took {:?}", pp_end);

src/cli/repl/mod.rs

@@ -337,7 +337,6 @@ where
                     let instance =
                         Instance::new(self.rc, self.lang.clone(), true, Kind::NovaPublicParams);
                     let pp = public_params(&instance)?;
-
                     let prover = NovaProver::<_, C>::new(self.rc, self.lang.clone());
 
                     info!("Proving");

src/proof/nova.rs

@@ -13,6 +13,7 @@ use nova::{
     },
     CompressedSNARK, ProverKey, R1CSWithArity, RecursiveSNARK, VerifierKey,
 };
+use once_cell::sync::OnceCell;
 use pasta_curves::pallas;
 use serde::{Deserialize, Serialize};
 use std::{
@@ -112,39 +113,54 @@ pub type NovaPublicParams<F, C1> = nova::PublicParams<E1<F>, E2<F>, C1, C2<F>>;
 pub struct PublicParams<F, SC: StepCircuit<F>>
 where
     F: CurveCycleEquipped,
-    // technical bounds that would disappear once associated_type_bounds stabilizes
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
 {
-    pp: NovaPublicParams<F, SC>,
-    pk: ProverKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>,
-    vk: VerifierKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>,
+    /// Public parameters for the Nova proving system.
+    pub pp: NovaPublicParams<F, SC>,
+    /// Prover and verifier key for final proof compression
+    #[serde(skip)]
+    pk_and_vk: OnceCell<(
+        ProverKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>,
+        VerifierKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>,
+    )>,
 }
 
-impl<F: CurveCycleEquipped, SC: StepCircuit<F>> Abomonation for PublicParams<F, SC>
-where
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
+// this avoids dipping into the pk/vk
+impl<F: CurveCycleEquipped, SC: StepCircuit<F> + std::fmt::Debug> std::fmt::Debug
+    for PublicParams<F, SC>
 {
-    unsafe fn entomb<W: std::io::Write>(&self, bytes: &mut W) -> std::io::Result<()> {
-        self.pp.entomb(bytes)?;
-        self.pk.entomb(bytes)?;
-        self.vk.entomb(bytes)?;
-        Ok(())
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PublicParams")
+            .field("pp", &self.pp)
+            .finish()
+    }
+}
+
+impl<F: CurveCycleEquipped, SC: StepCircuit<F>> PublicParams<F, SC> {
+    /// provides a reference to a ProverKey suitable for producing a CompressedProof
+    pub fn pk(&self) -> &ProverKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>> {
+        let (pk, _vk) = self.pk_and_vk.get_or_init(|| {
+            CompressedSNARK::<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>::setup(&self.pp).unwrap()
+        });
+        pk
     }
 
-    unsafe fn exhume<'b>(&mut self, mut bytes: &'b mut [u8]) -> Option<&'b mut [u8]> {
-        let temp = bytes;
-        bytes = self.pp.exhume(temp)?;
-        let temp = bytes;
-        bytes = self.pk.exhume(temp)?;
-        let temp = bytes;
-        bytes = self.vk.exhume(temp)?;
-        Some(bytes)
+    /// provides a reference to a VerifierKey suitable for verifying a CompressedProof
+    pub fn vk(&self) -> &VerifierKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>> {
+        let (_pk, vk) = self.pk_and_vk.get_or_init(|| {
+            CompressedSNARK::<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>::setup(&self.pp).unwrap()
+        });
+        vk
     }
+}
 
-    fn extent(&self) -> usize {
-        self.pp.extent() + self.pk.extent() + self.vk.extent()
+impl<F: CurveCycleEquipped, SC: StepCircuit<F>> From<NovaPublicParams<F, SC>>
+    for PublicParams<F, SC>
+{
+    fn from(pp: NovaPublicParams<F, SC>) -> PublicParams<F, SC> {
+        PublicParams {
+            pp,
+            pk_and_vk: OnceCell::new(),
+        }
     }
 }
 
@@ -202,8 +218,10 @@ where
         &*commitment_size_hint1,
         &*commitment_size_hint2,
     );
-    let (pk, vk) = CompressedSNARK::setup(&pp).unwrap();
-    PublicParams { pp, pk, vk }
+    PublicParams {
+        pp,
+        pk_and_vk: OnceCell::new(),
+    }
 }
 
 /// Generates the circuits for the Nova proving system.
@@ -342,7 +360,7 @@ where
             Self::Recursive(recursive_snark, num_steps) => Ok(Self::Compressed(
                 Box::new(CompressedSNARK::<_, _, _, _, SS1<F>, SS2<F>>::prove(
                     &pp.pp,
-                    &pp.pk,
+                    pp.pk(),
                     &recursive_snark,
                 )?),
                 num_steps,
@@ -361,7 +379,7 @@ where
                 p.verify(&pp.pp, *num_steps, z0_primary, &z0_secondary)?
             }
             Self::Compressed(p, num_steps) => {
-                p.verify(&pp.vk, *num_steps, z0_primary, &z0_secondary)?
+                p.verify(pp.vk(), *num_steps, z0_primary, &z0_secondary)?
             }
         };
 

src/public_parameters/disk_cache.rs

@@ -1,6 +1,7 @@
 use std::fs::create_dir_all;
 use std::io::{BufReader, BufWriter, Read};
 use std::marker::PhantomData;
+use std::sync::Arc;
 
 use abomonation::{encode, Abomonation};
 use camino::{Utf8Path, Utf8PathBuf};
@@ -19,16 +20,16 @@ pub(crate) fn public_params_dir() -> &'static Utf8PathBuf {
     &lurk_config(None, None).public_params_dir
 }
 
-pub(crate) struct DiskCache<'a, F, C>
+pub(crate) struct DiskCache<F, C>
 where
     F: CurveCycleEquipped,
-    C: Coprocessor<F> + 'a,
+    C: Coprocessor<F>,
 {
     dir: Utf8PathBuf,
-    _t: PhantomData<(&'a (), F, C)>,
+    _t: PhantomData<(F, C)>,
 }
 
-impl<'a, F: CurveCycleEquipped, C: Coprocessor<F> + 'a> DiskCache<'a, F, C>
+impl<F: CurveCycleEquipped, C: Coprocessor<F>> DiskCache<F, C>
 where
     // technical bounds that would disappear once associated_type_bounds stabilizes
     <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
@@ -43,10 +44,10 @@ where
         })
     }
 
-    pub(crate) fn read(
+    pub(crate) fn read<'a>(
         &self,
-        instance: &Instance<'a, F, C>,
-    ) -> Result<PublicParams<F, C1LEM<'a, F, C>>, Error> {
+        instance: &Instance<F, C>,
+    ) -> Result<Arc<PublicParams<F, C1LEM<'a, F, C>>>, Error> {
         let file = instance.open(&self.dir)?;
         let reader = BufReader::new(file);
         bincode::deserialize_from(reader)
@@ -55,7 +56,7 @@ where
 
     pub(crate) fn read_bytes(
         &self,
-        instance: &Instance<'a, F, C>,
+        instance: &Instance<F, C>,
         byte_sink: &mut Vec<u8>,
     ) -> Result<(), Error> {
         let file = instance.open(&self.dir)?;
@@ -66,18 +67,18 @@ where
 
     pub(crate) fn write(
         &self,
-        instance: &Instance<'a, F, C>,
-        data: &PublicParams<F, C1LEM<'a, F, C>>,
+        instance: &Instance<F, C>,
+        data: &Arc<PublicParams<F, C1LEM<'_, F, C>>>,
     ) -> Result<(), Error> {
         let file = instance.create(&self.dir)?;
         let writer = BufWriter::new(&file);
-        bincode::serialize_into(writer, &data)
+        bincode::serialize_into(writer, data)
             .map_err(|e| Error::Cache(format!("Public param cache serialization error: {}", e)))
     }
 
     pub(crate) fn write_abomonated<V: Abomonation>(
         &self,
-        instance: &Instance<'a, F, C>,
+        instance: &Instance<F, C>,
         data: &V,
     ) -> Result<(), Error> {
         let mut file = instance.create(&self.dir)?;