feat: retrieve subnet contracts from docker image

In order to make this work, this changeset also includes the ability to
track a cached contract's clarity version, so that it can properly be
deployed with the correct version.

components/clarinet-cli/src/frontend/cli.rs

@@ -11,8 +11,7 @@ use crate::generate::{
 };
 use crate::integrate;
 use crate::lsp::run_lsp;
-use crate::runner::run_scripts;
-use crate::runner::DeploymentCache;
+use crate::runner::{block_on, run_scripts, DeploymentCache};
 use chainhook_types::StacksNetwork;
 use chainhook_types::{BitcoinNetwork, Chain};
 use clarinet_deployments::onchain::{
@@ -36,11 +35,11 @@ use clarity_repl::repl::diagnostic::{output_code, output_diagnostic};
 use clarity_repl::repl::{ClarityCodeSource, ClarityContract, ContractDeployer, DEFAULT_EPOCH};
 use clarity_repl::{analysis, repl, Terminal};
 use stacks_network::chainhook_event_observer::chainhooks::types::ChainhookSpecification;
+use stacks_network::chainhook_event_observer::utils::Context;
 use stacks_network::{self, DevnetOrchestrator};
 use std::collections::HashMap;
 use std::fs::{self, File};
 use std::io::prelude::*;
-use std::path::PathBuf;
 use std::{env, process};
 
 use clap::{IntoApp, Parser, Subcommand};
@@ -1265,6 +1264,64 @@ pub fn main() {
         Command::Integrate(cmd) => {
             let manifest = load_manifest_or_exit(cmd.manifest_path);
             println!("Computing deployment plan");
+
+            let mut orchestrator = match DevnetOrchestrator::new(manifest.clone(), None) {
+                Ok(orchestrator) => orchestrator,
+                Err(e) => {
+                    println!("{}", format_err!(e));
+                    process::exit(1);
+                }
+            };
+
+            let cache_location =
+                if let FileLocation::FileSystem { path } = &manifest.project.cache_location {
+                    path.clone()
+                } else {
+                    println!("cache location must be a file system path");
+                    process::exit(1);
+                };
+
+            let devnet_config = orchestrator
+                .network_config
+                .as_ref()
+                .and_then(|c| c.devnet.as_ref())
+                .expect("devnet configuration not found");
+
+            let _ = fs::create_dir(cache_location.clone());
+            let _ = fs::create_dir(devnet_config.working_dir.clone());
+            let _ = fs::create_dir(format!("{}/conf", devnet_config.working_dir));
+            let _ = fs::create_dir(format!("{}/data", devnet_config.working_dir));
+
+            if devnet_config.enable_subnet_node {
+                let subnet_deployer =
+                    match QualifiedContractIdentifier::parse(&devnet_config.subnet_contract_id) {
+                        Ok(contract) => contract,
+                        Err(e) => {
+                            println!("invalid subnet contract id: {}", e);
+                            process::exit(1);
+                        }
+                    }
+                    .issuer;
+
+                let _ = fs::create_dir(format!("{}/requirements", cache_location.display()));
+
+                let ctx = Context {
+                    logger: None,
+                    tracer: false,
+                };
+                match block_on(orchestrator.prepare_subnet_node_container(
+                    1,
+                    &ctx,
+                    cache_location,
+                    &subnet_deployer,
+                )) {
+                    Ok(_) => {}
+                    Err(e) => {
+                        process::exit(1);
+                    }
+                };
+            }
+
             let result = match cmd.deployment_plan_path {
                 None => {
                     let res = load_deployment_if_exists(
@@ -1329,14 +1386,6 @@ pub fn main() {
                 }
             };
 
-            let orchestrator = match DevnetOrchestrator::new(manifest, None) {
-                Ok(orchestrator) => orchestrator,
-                Err(e) => {
-                    println!("{}", format_err!(e));
-                    process::exit(1);
-                }
-            };
-
             if orchestrator.manifest.project.telemetry {
                 #[cfg(feature = "telemetry")]
                 telemetry_report_event(DeveloperUsageEvent::DevnetExecuted(

components/clarinet-cli/src/integrate/mod.rs

@@ -1,5 +1,4 @@
 use std::{
-    fs,
     path::PathBuf,
     str::FromStr,
     sync::mpsc::{self, channel, Sender},
@@ -46,8 +45,6 @@ pub fn run_devnet(
         .and_then(|c| c.devnet.as_ref())
         .and_then(|d| Some(d.working_dir.to_string()))
         .ok_or("unable to read settings/Devnet.toml")?;
-    fs::create_dir_all(&working_dir)
-        .map_err(|_| format!("unable to create dir {}", working_dir))?;
     let mut log_path = PathBuf::from_str(&working_dir)
         .map_err(|e| format!("unable to working_dir {}\n{}", working_dir, e.to_string()))?;
     log_path.push("devnet.log");

components/clarinet-deployments/src/lib.rs

@@ -1,7 +1,7 @@
 use clarity_repl::clarity::stacks_common::types::StacksEpochId;
 use clarity_repl::clarity::ClarityVersion;
-use clarity_repl::repl::DEFAULT_EPOCH;
 use clarity_repl::repl::{ClarityCodeSource, ClarityContract, ContractDeployer};
+use clarity_repl::repl::{DEFAULT_CLARITY_VERSION, DEFAULT_EPOCH};
 
 extern crate serde;
 
@@ -322,7 +322,11 @@ pub async fn generate_default_deployment(
                     ))
                 }
             };
-            queue.push_front(contract_id)
+            queue.push_front((
+                contract_id,
+                StacksEpochId::Epoch21,
+                ClarityVersion::Clarity2,
+            ));
         }
     }
 
@@ -331,9 +335,11 @@ pub async fn generate_default_deployment(
         let cache_location = &manifest.project.cache_location;
         let mut emulated_contracts_publish = HashMap::new();
         let mut requirements_publish = HashMap::new();
+        let mut contract_epochs = HashMap::new();
         // TODO: This is fine for now, but will need to be changed after 2.1 is live and
         //       there are requirements that need to be deployed in epoch 2.1.
-        let requirements_epoch = EpochSpec::Epoch2_0;
+        let requirements_epoch = DEFAULT_EPOCH;
+        let clarity_version = DEFAULT_CLARITY_VERSION;
 
         // Load all the requirements
         // Some requirements are explicitly listed, some are discovered as we compute the ASTs.
@@ -348,10 +354,14 @@ pub async fn generate_default_deployment(
                     ))
                 }
             };
-            queue.push_front(contract_id);
+            queue.push_front((
+                contract_id,
+                requirements_epoch.clone(),
+                clarity_version.clone(),
+            ));
         }
 
-        while let Some(contract_id) = queue.pop_front() {
+        while let Some((contract_id, epoch, clarity_version)) = queue.pop_front() {
             // Extract principal from contract_id
             if requirements_deps.contains_key(&contract_id) {
                 continue;
@@ -362,7 +372,7 @@ pub async fn generate_default_deployment(
                 Some(ast) => ast,
                 None => {
                     // Download the code
-                    let (source, clarity_version, contract_location) =
+                    let (source, epoch, clarity_version, contract_location) =
                         requirements::retrieve_contract(
                             &contract_id,
                             &cache_location,
@@ -377,7 +387,7 @@ pub async fn generate_default_deployment(
                             emulated_sender: contract_id.issuer.clone(),
                             source: source.clone(),
                             location: contract_location,
-                            clarity_version: ClarityVersion::Clarity1,
+                            clarity_version: clarity_version.clone(),
                         };
                         emulated_contracts_publish.insert(contract_id.clone(), data);
                     } else if network.either_devnet_or_testnet() {
@@ -408,15 +418,16 @@ pub async fn generate_default_deployment(
                             clarity_version,
                         };
                         requirements_publish.insert(contract_id.clone(), data);
+                        contract_epochs.insert(contract_id.clone(), epoch);
                     }
 
                     // Compute the AST
                     let contract = ClarityContract {
                         code_source: ClarityCodeSource::ContractInMemory(source),
                         name: contract_id.name.to_string(),
                         deployer: ContractDeployer::ContractIdentifier(contract_id.clone()),
-                        clarity_version: ClarityVersion::Clarity1,
-                        epoch: forced_epoch.unwrap_or(DEFAULT_EPOCH),
+                        clarity_version: clarity_version,
+                        epoch: forced_epoch.unwrap_or(epoch),
                     };
                     let (ast, _, _) = session.interpreter.build_ast(&contract);
                     ast
@@ -439,7 +450,11 @@ pub async fn generate_default_deployment(
                     // result in the `inferable_dependencies` map. We will just extract and keep the associated data (source, ast, deps).
                     for (contract_id, dependencies) in inferable_dependencies.into_iter() {
                         for dependency in dependencies.iter() {
-                            queue.push_back(dependency.contract_id.clone());
+                            queue.push_back((
+                                dependency.contract_id.clone(),
+                                epoch.clone(),
+                                clarity_version.clone(),
+                            ));
                         }
                         requirements_deps.insert(contract_id.clone(), dependencies);
                         requirements_asts.insert(contract_id.clone(), ast);
@@ -452,14 +467,18 @@ pub async fn generate_default_deployment(
                     // and we will keep the source in memory to avoid useless disk access.
                     for (_, dependencies) in inferable_dependencies.iter() {
                         for dependency in dependencies.iter() {
-                            queue.push_back(dependency.contract_id.clone());
+                            queue.push_back((
+                                dependency.contract_id.clone(),
+                                epoch.clone(),
+                                clarity_version.clone(),
+                            ));
                         }
                     }
                     requirements_asts.insert(contract_id.clone(), ast);
-                    queue.push_front(contract_id);
+                    queue.push_front((contract_id, epoch, clarity_version));
 
                     for non_inferable_contract_id in non_inferable_dependencies.into_iter() {
-                        queue.push_front(non_inferable_contract_id);
+                        queue.push_front((non_inferable_contract_id, epoch, clarity_version));
                     }
                 }
             };
@@ -482,15 +501,23 @@ pub async fn generate_default_deployment(
                         .remove(contract_id)
                         .expect("unable to retrieve contract");
                     let tx = TransactionSpecification::EmulatedContractPublish(data);
-                    add_transaction_to_epoch(&mut transactions, tx, &requirements_epoch);
+                    add_transaction_to_epoch(
+                        &mut transactions,
+                        tx,
+                        &contract_epochs[contract_id].into(),
+                    );
                 }
             } else if network.either_devnet_or_testnet() {
                 for contract_id in ordered_contracts_ids.iter() {
                     let data = requirements_publish
                         .remove(contract_id)
                         .expect("unable to retrieve contract");
                     let tx = TransactionSpecification::RequirementPublish(data);
-                    add_transaction_to_epoch(&mut transactions, tx, &requirements_epoch);
+                    add_transaction_to_epoch(
+                        &mut transactions,
+                        tx,
+                        &contract_epochs[contract_id].into(),
+                    );
                 }
             }
         }

components/clarinet-deployments/src/onchain/mod.rs

@@ -666,9 +666,18 @@ pub fn apply_on_chain_deployment(
 
                     // Remapping principals - This is happening
                     let mut source = tx.source.clone();
-                    for (src_principal, dst_principal) in tx.remap_principals.iter() {
-                        let src = src_principal.to_address();
-                        let dst = dst_principal.to_address();
+                    for (src_principal, dst_principal) in tx
+                        .remap_principals
+                        .iter()
+                        .map(|(src, dst)| (src.to_address(), dst.to_address()))
+                        .chain(
+                            contracts_ids_to_remap
+                                .iter()
+                                .map(|(k, v)| (k.clone(), v.clone())),
+                        )
+                    {
+                        let src = src_principal;
+                        let dst = dst_principal;
                         let mut matched_indices = source
                             .match_indices(&src)
                             .map(|(i, _)| i)