Add s3 credential support (#1606)

Author: eandre

Cargo.lock

@@ -384,8 +384,10 @@ func (c *legacyConverter) Convert() (*config.Runtime, error) {
 				switch prov := cluster.Provider.(type) {
 				case *runtimev1.BucketCluster_S3_:
 					p.S3 = &config.S3BucketProvider{
-						Region:   prov.S3.GetRegion(),
-						Endpoint: prov.S3.Endpoint,
+						Region:          prov.S3.GetRegion(),
+						Endpoint:        prov.S3.Endpoint,
+						AccessKeyID:     prov.S3.AccessKeyId,
+						SecretAccessKey: ptrOrNil(c.secretString(prov.S3.SecretAccessKey)),
 					}
 				case *runtimev1.BucketCluster_Gcs:
 					p.GCS = &config.GCSBucketProvider{
@@ -640,6 +642,14 @@ func ptr[T comparable](val T) *T {
 	return &val
 }
 
+func ptrOrNil[T comparable](val T) *T {
+	var zero T
+	if val == zero {
+		return nil
+	}
+	return &val
+}
+
 func randomMapValue[K comparable, V any](m map[K]V) (V, bool) {
 	for _, v := range m {
 		return v, true

pkg/rtconfgen/convert.go

@@ -323,6 +323,11 @@ message BucketCluster {
 
     // Endpoint override, if any. Must be specified if using a non-standard AWS region.
     optional string endpoint = 2;
+
+    // Set these to use explicit credentials for this bucket,
+    // as opposed to resolving using AWS's default credential chain.
+    optional string access_key_id = 3;
+    optional SecretData secret_access_key = 4;
   }
 
   message GCS {

proto/encore/runtime/v1/infra.pb.go

@@ -93,6 +93,7 @@ md5 = "0.7.0"
 aws-sdk-s3 = "1.58.0"
 aws-smithy-types = { version = "1.2.8", features = ["byte-stream-poll-next", "rt-tokio"] }
 percent-encoding = "2.3.1"
+aws-credential-types = "1.2.1"
 
 [build-dependencies]
 prost-build = "0.12.3"

proto/encore/runtime/v1/infra.proto

@@ -46,6 +46,8 @@ pub struct GCS {
 pub struct S3 {
     pub region: String,
     pub endpoint: Option<String>,
+    pub access_key_id: Option<String>,
+    pub secret_access_key: Option<EnvString>,
     pub buckets: HashMap<String, Bucket>,
 }
 
@@ -461,6 +463,11 @@ pub fn map_infra_to_runtime(infra: InfraConfig) -> RuntimeConfig {
                         pbruntime::bucket_cluster::S3 {
                             region: s3.region,
                             endpoint: s3.endpoint,
+                            access_key_id: s3.access_key_id,
+                            secret_access_key: s3
+                                .secret_access_key
+                                .as_ref()
+                                .map(map_env_string_to_secret_data),
                         },
                     )),
                     buckets: s3

runtimes/core/Cargo.toml

@@ -331,7 +331,8 @@ impl Runtime {
             .context("failed to resolve gateway push subscriptions")?;
 
         let pubsub = pubsub::Manager::new(tracer.clone(), resources.pubsub_clusters, &md)?;
-        let objects = objects::Manager::new(tracer.clone(), resources.bucket_clusters, &md);
+        let objects =
+            objects::Manager::new(&secrets, tracer.clone(), resources.bucket_clusters, &md);
         let sqldb = sqldb::ManagerConfig {
             clusters: resources.sql_clusters,
             creds: &creds,

runtimes/core/src/infracfg.rs

@@ -363,7 +363,7 @@ impl objects::ObjectImpl for Object {
             return Err(PublicUrlError::PrivateBucket);
         };
 
-        let url = objects::public_url(base_url, self.bkt.key_prefix.as_deref(), &self.key);
+        let url = objects::public_url(base_url, &self.key);
         Ok(url)
     }
 }

runtimes/core/src/lib.rs

@@ -5,6 +5,7 @@ use crate::encore::parser::meta::v1 as meta;
 use crate::encore::runtime::v1 as pb;
 use crate::names::EncoreName;
 use crate::objects::{gcs, noop, s3, BucketImpl, ClusterImpl};
+use crate::secrets;
 use crate::trace::Tracer;
 
 use super::Bucket;
@@ -17,8 +18,13 @@ pub struct Manager {
 }
 
 impl Manager {
-    pub fn new(tracer: Tracer, clusters: Vec<pb::BucketCluster>, md: &meta::Data) -> Self {
-        let bucket_cfg = make_cfg_maps(clusters, md);
+    pub fn new(
+        secrets: &secrets::Manager,
+        tracer: Tracer,
+        clusters: Vec<pb::BucketCluster>,
+        md: &meta::Data,
+    ) -> Self {
+        let bucket_cfg = make_cfg_maps(secrets, clusters, md);
 
         Self {
             tracer,
@@ -54,13 +60,20 @@ impl Manager {
 }
 
 fn make_cfg_maps(
+    secrets: &secrets::Manager,
     clusters: Vec<pb::BucketCluster>,
     _md: &meta::Data,
 ) -> HashMap<EncoreName, (Arc<dyn ClusterImpl>, pb::Bucket)> {
     let mut bucket_map = HashMap::new();
 
     for cluster_cfg in clusters {
-        let cluster = new_cluster(&cluster_cfg);
+        let cluster = match cluster_cfg.provider {
+            Some(provider) => new_cluster(secrets, provider),
+            None => {
+                log::error!("missing bucket cluster provider: {}", cluster_cfg.rid);
+                Arc::new(noop::Cluster)
+            }
+        };
 
         for bucket_cfg in cluster_cfg.buckets {
             bucket_map.insert(
@@ -73,14 +86,18 @@ fn make_cfg_maps(
     bucket_map
 }
 
-fn new_cluster(cluster: &pb::BucketCluster) -> Arc<dyn ClusterImpl> {
-    let Some(provider) = &cluster.provider else {
-        log::error!("missing bucket cluster provider: {}", cluster.rid);
-        return Arc::new(noop::Cluster);
-    };
-
+fn new_cluster(
+    secrets: &secrets::Manager,
+    provider: pb::bucket_cluster::Provider,
+) -> Arc<dyn ClusterImpl> {
     match provider {
-        pb::bucket_cluster::Provider::S3(s3cfg) => Arc::new(s3::Cluster::new(s3cfg)),
+        pb::bucket_cluster::Provider::S3(s3cfg) => {
+            let secret_access_key = s3cfg
+                .secret_access_key
+                .as_ref()
+                .map(|k| secrets.load(k.clone()));
+            Arc::new(s3::Cluster::new(s3cfg, secret_access_key))
+        }
         pb::bucket_cluster::Provider::Gcs(gcscfg) => Arc::new(gcs::Cluster::new(gcscfg.clone())),
     }
 }

runtimes/core/src/objects/gcs/bucket.rs

@@ -493,16 +493,13 @@ fn escape_path(s: &str) -> Cow<'_, str> {
     percent_encoding::percent_encode(s.as_bytes(), PATH).into()
 }
 
-/// Computes the public url given a base url, optional key prefix, and object name.
-fn public_url(base_url: String, key_prefix: Option<&str>, name: &str) -> String {
+/// Computes the public url given a base url and object name.
+fn public_url(base_url: String, name: &str) -> String {
     let mut url = base_url;
 
     if !url.ends_with('/') {
         url.push('/');
     }
-    if let Some(key_prefix) = key_prefix {
-        url.push_str(&escape_path(key_prefix));
-    }
     url.push_str(&escape_path(name));
     url
 }