Feature/eng 486 update deployer with runtime changes

.circleci/config.yml

@@ -89,6 +89,7 @@ commands:
             shuttle-aws-rds = { path = "$PWD/resources/aws-rds" }
             shuttle-persist = { path = "$PWD/resources/persist" }
             shuttle-next = { path = "$PWD/next" }
+            shuttle-runtime = { path = "$PWD/runtime" }
             shuttle-shared-db = { path = "$PWD/resources/shared-db" }
             shuttle-secrets = { path = "$PWD/resources/secrets" }
             shuttle-static-folder = { path = "$PWD/resources/static-folder" }

cargo-shuttle/src/lib.rs

@@ -449,7 +449,7 @@ impl Shuttle {
 
         let service_name = self.ctx.project_name().to_string();
 
-        let (is_wasm, bin_path) = match runtime {
+        let (is_wasm, executable_path) = match runtime {
             Runtime::Next(path) => (true, path),
             Runtime::Legacy(path) => (false, path),
         };
@@ -479,6 +479,8 @@ impl Shuttle {
                         .arg(path)
                         .arg("--bin")
                         .arg("shuttle-next")
+                        .arg("--features")
+                        .arg("next")
                         .output()
                         .expect("failed to install the shuttle runtime");
                 } else {
@@ -499,14 +501,16 @@ impl Shuttle {
                             .arg("https://github.com/shuttle-hq/shuttle")
                             .arg("--branch")
                             .arg("production")
+                            .arg("--features")
+                            .arg("next")
                             .output()
                             .expect("failed to install the shuttle runtime");
                     };
                 };
 
                 runtime_path
             } else {
-                bin_path.clone()
+                executable_path.clone()
             }
         };
 
@@ -525,7 +529,7 @@ impl Shuttle {
         })?;
 
         let load_request = tonic::Request::new(LoadRequest {
-            path: bin_path
+            path: executable_path
                 .into_os_string()
                 .into_string()
                 .expect("to convert path to string"),

codegen/src/shuttle_main/mod.rs

@@ -239,8 +239,8 @@ impl ToTokens for Loader {
         };
 
         let loader = quote! {
-            async fn loader<S: shuttle_runtime::StorageManager>(
-                mut #factory_ident: shuttle_runtime::ProvisionerFactory<S>,
+            async fn loader(
+                mut #factory_ident: shuttle_runtime::ProvisionerFactory,
                 logger: shuttle_runtime::Logger,
             ) -> #return_type {
                 use shuttle_service::Context;
@@ -299,8 +299,8 @@ mod tests {
 
         let actual = quote!(#input);
         let expected = quote! {
-            async fn loader<S: shuttle_runtime::StorageManager>(
-                mut _factory: shuttle_runtime::ProvisionerFactory<S>,
+            async fn loader(
+                mut _factory: shuttle_runtime::ProvisionerFactory,
                 logger: shuttle_runtime::Logger,
             ) -> ShuttleSimple {
                 use shuttle_service::Context;
@@ -379,8 +379,8 @@ mod tests {
 
         let actual = quote!(#input);
         let expected = quote! {
-            async fn loader<S: shuttle_runtime::StorageManager>(
-                mut factory: shuttle_runtime::ProvisionerFactory<S>,
+            async fn loader(
+                mut factory: shuttle_runtime::ProvisionerFactory,
                 logger: shuttle_runtime::Logger,
             ) -> ShuttleComplex {
                 use shuttle_service::Context;
@@ -494,8 +494,8 @@ mod tests {
 
         let actual = quote!(#input);
         let expected = quote! {
-            async fn loader<S: shuttle_runtime::StorageManager>(
-                mut factory: shuttle_runtime::ProvisionerFactory<S>,
+            async fn loader(
+                mut factory: shuttle_runtime::ProvisionerFactory,
                 logger: shuttle_runtime::Logger,
             ) -> ShuttleComplex {
                 use shuttle_service::Context;

common/src/storage_manager.rs

@@ -2,14 +2,14 @@ use std::{fs, io, path::PathBuf};
 
 use uuid::Uuid;
 
-pub trait StorageManager: Clone + Sync + Send {
+pub trait StorageManager: Sync + Send {
     /// Path for a specific service build files
-    fn service_build_path<S: AsRef<str>>(&self, service_name: S) -> Result<PathBuf, io::Error>;
+    fn service_build_path(&self, service_name: &str) -> Result<PathBuf, io::Error>;
 
     /// Path to folder for storing deployment files
-    fn deployment_storage_path<S: AsRef<str>>(
+    fn deployment_storage_path(
         &self,
-        service_name: S,
+        service_name: &str,
         deployment_id: &Uuid,
     ) -> Result<PathBuf, io::Error>;
 }
@@ -33,19 +33,19 @@ impl ArtifactsStorageManager {
         Ok(builds_path)
     }
 
-    /// The directory in which compiled '.so' files are stored.
-    pub fn libraries_path(&self) -> Result<PathBuf, io::Error> {
-        let libs_path = self.artifacts_path.join("shuttle-libs");
-        fs::create_dir_all(&libs_path)?;
+    /// The directory in which compiled executables are stored.
+    pub fn executables_path(&self) -> Result<PathBuf, io::Error> {
+        let executables_path = self.artifacts_path.join("shuttle-executables");
+        fs::create_dir_all(&executables_path)?;
 
-        Ok(libs_path)
+        Ok(executables_path)
     }
 
-    /// Path to `.so` for a service
-    pub fn deployment_library_path(&self, deployment_id: &Uuid) -> Result<PathBuf, io::Error> {
-        let library_path = self.libraries_path()?.join(deployment_id.to_string());
+    /// Path to executable for a service
+    pub fn deployment_executable_path(&self, deployment_id: &Uuid) -> Result<PathBuf, io::Error> {
+        let executable_path = self.executables_path()?.join(deployment_id.to_string());
 
-        Ok(library_path)
+        Ok(executable_path)
     }
 
     /// Path of the directory to store user files
@@ -58,21 +58,21 @@ impl ArtifactsStorageManager {
 }
 
 impl StorageManager for ArtifactsStorageManager {
-    fn service_build_path<S: AsRef<str>>(&self, service_name: S) -> Result<PathBuf, io::Error> {
-        let builds_path = self.builds_path()?.join(service_name.as_ref());
+    fn service_build_path(&self, service_name: &str) -> Result<PathBuf, io::Error> {
+        let builds_path = self.builds_path()?.join(service_name);
         fs::create_dir_all(&builds_path)?;
 
         Ok(builds_path)
     }
 
-    fn deployment_storage_path<S: AsRef<str>>(
+    fn deployment_storage_path(
         &self,
-        service_name: S,
+        service_name: &str,
         deployment_id: &Uuid,
     ) -> Result<PathBuf, io::Error> {
         let storage_path = self
             .storage_path()?
-            .join(service_name.as_ref())
+            .join(service_name)
             .join(deployment_id.to_string());
         fs::create_dir_all(&storage_path)?;
 
@@ -93,13 +93,13 @@ impl WorkingDirStorageManager {
 }
 
 impl StorageManager for WorkingDirStorageManager {
-    fn service_build_path<S: AsRef<str>>(&self, _service_name: S) -> Result<PathBuf, io::Error> {
+    fn service_build_path(&self, _service_name: &str) -> Result<PathBuf, io::Error> {
         Ok(self.working_dir.clone())
     }
 
-    fn deployment_storage_path<S: AsRef<str>>(
+    fn deployment_storage_path(
         &self,
-        _service_name: S,
+        _service_name: &str,
         _deployment_id: &Uuid,
     ) -> Result<PathBuf, io::Error> {
         Ok(self.working_dir.clone())

deployer/prepare.sh

@@ -9,17 +9,18 @@
 mkdir -p $CARGO_HOME; \
 echo '[patch.crates-io]
 shuttle-service = { path = "/usr/src/shuttle/service" }
+shuttle-runtime = { path = "/usr/src/shuttle/runtime" }
 shuttle-aws-rds = { path = "/usr/src/shuttle/resources/aws-rds" }
 shuttle-persist = { path = "/usr/src/shuttle/resources/persist" }
 shuttle-shared-db = { path = "/usr/src/shuttle/resources/shared-db" }
 shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" }
 shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }' > $CARGO_HOME/config.toml
-#
+
 # Add the wasm32-wasi target
 rustup target add wasm32-wasi
 
 # Install the shuttle runtime
-cargo install shuttle-runtime --path "/usr/src/shuttle/runtime"
+cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" --bin shuttle-next --features next
 
 while getopts "p," o; do
     case $o in

deployer/src/deployment/queue.rs

@@ -224,9 +224,9 @@ impl Queued {
             run_pre_deploy_tests(&project_path, tx).await?;
         }
 
-        info!("Moving built library");
+        info!("Moving built executable");
 
-        store_lib(&storage_manager, &runtime, &self.id).await?;
+        store_executable(&storage_manager, &runtime, &self.id).await?;
 
         let is_next = matches!(runtime, Runtime::Next(_));
 
@@ -396,21 +396,22 @@ async fn run_pre_deploy_tests(
     }
 }
 
-/// Store 'so' file in the libs folder
+/// This will store the path to the executable for each runtime, which will be the users project with
+/// an embedded runtime for legacy, and a .wasm file for shuttle-next.
 #[instrument(skip(storage_manager, runtime, id))]
-async fn store_lib(
+async fn store_executable(
     storage_manager: &ArtifactsStorageManager,
     runtime: &Runtime,
     id: &Uuid,
 ) -> Result<()> {
-    let so_path = match runtime {
+    let executable_path = match runtime {
         Runtime::Next(path) => path,
         Runtime::Legacy(path) => path,
     };
 
-    let new_so_path = storage_manager.deployment_library_path(id)?;
+    let new_executable_path = storage_manager.deployment_executable_path(id)?;
 
-    fs::rename(so_path, new_so_path).await?;
+    fs::rename(executable_path, new_executable_path).await?;
 
     Ok(())
 }
@@ -550,30 +551,34 @@ ff0e55bda1ff01000000000000000000e0079c01ff12a55500280000",
     }
 
     #[tokio::test]
-    async fn store_lib() {
-        let libs_dir = Builder::new().prefix("lib-store").tempdir().unwrap();
-        let libs_p = libs_dir.path();
-        let storage_manager = ArtifactsStorageManager::new(libs_p.to_path_buf());
+    async fn store_executable() {
+        let executables_dir = Builder::new().prefix("executable-store").tempdir().unwrap();
+        let executables_p = executables_dir.path();
+        let storage_manager = ArtifactsStorageManager::new(executables_p.to_path_buf());
 
         let build_p = storage_manager.builds_path().unwrap();
 
-        let so_path = build_p.join("xyz.so");
-        let runtime = Runtime::Legacy(so_path.clone());
+        let executable_path = build_p.join("xyz");
+        let runtime = Runtime::Legacy(executable_path.clone());
         let id = Uuid::new_v4();
 
-        fs::write(&so_path, "barfoo").await.unwrap();
+        fs::write(&executable_path, "barfoo").await.unwrap();
 
-        super::store_lib(&storage_manager, &runtime, &id)
+        super::store_executable(&storage_manager, &runtime, &id)
             .await
             .unwrap();
 
-        // Old '.so' file gone?
-        assert!(!so_path.exists());
+        // Old executable file gone?
+        assert!(!executable_path.exists());
 
         assert_eq!(
-            fs::read_to_string(libs_p.join("shuttle-libs").join(id.to_string()))
-                .await
-                .unwrap(),
+            fs::read_to_string(
+                executables_p
+                    .join("shuttle-executables")
+                    .join(id.to_string())
+            )
+            .await
+            .unwrap(),
             "barfoo"
         );
     }

deployer/src/deployment/run.rs

@@ -182,7 +182,10 @@ impl Built {
         kill_old_deployments: impl futures::Future<Output = Result<()>>,
         cleanup: impl FnOnce(SubscribeStopResponse) + Send + 'static,
     ) -> Result<()> {
-        let so_path = storage_manager.deployment_library_path(&self.id)?;
+        // For legacy this is the path to the users project with an embedded runtime.
+        // For shuttle-next this is the path to the compiled .wasm file, which will be
+        // used in the load request.
+        let executable_path = storage_manager.deployment_executable_path(&self.id)?;
 
         let port = match pick_unused_port() {
             Some(port) => port,
@@ -194,10 +197,18 @@ impl Built {
         };
 
         let address = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), port);
+
+        let legacy_runtime_path = if self.is_next {
+            // The runtime client for next is the installed shuttle-next bin
+            None
+        } else {
+            Some(executable_path.clone())
+        };
+
         let runtime_client = runtime_manager
             .lock()
             .await
-            .get_runtime_client(self.is_next)
+            .get_runtime_client(legacy_runtime_path.clone())
             .await
             .map_err(Error::Runtime)?;
 
@@ -208,7 +219,7 @@ impl Built {
         load(
             self.service_name.clone(),
             self.service_id,
-            so_path,
+            executable_path.clone(),
             secret_getter,
             runtime_client.clone(),
         )
@@ -230,13 +241,13 @@ impl Built {
 async fn load(
     service_name: String,
     service_id: Uuid,
-    so_path: PathBuf,
+    executable_path: PathBuf,
     secret_getter: impl SecretGetter,
     mut runtime_client: RuntimeClient<Channel>,
 ) -> Result<()> {
     info!(
         "loading project from: {}",
-        so_path
+        executable_path
             .clone()
             .into_os_string()
             .into_string()
@@ -252,7 +263,10 @@ async fn load(
     let secrets = HashMap::from_iter(secrets);
 
     let load_request = tonic::Request::new(LoadRequest {
-        path: so_path.into_os_string().into_string().unwrap_or_default(),
+        path: executable_path
+            .into_os_string()
+            .into_string()
+            .unwrap_or_default(),
         service_name: service_name.clone(),
         secrets,
     });
@@ -662,10 +676,9 @@ mod tests {
         let id = Uuid::new_v4();
         let so_path = crate_dir.join("target/release").join(lib_name);
         let storage_manager = get_storage_manager();
-        let new_so_path = storage_manager.deployment_library_path(&id).unwrap();
+        let new_so_path = storage_manager.deployment_executable_path(&id).unwrap();
 
         std::fs::copy(so_path, new_so_path).unwrap();
-
         (
             Built {
                 id,