Merge branch 'main' into feat/custom-telemetry-callbacks

Signed-off-by: aws-hans-pistor <133711035+aws-hans-pistor@users.noreply.github.com>

mountpoint-s3-client/CHANGELOG.md

@@ -1,5 +1,23 @@
 ## Unreleased
 
+### Other changes
+
+* Add parameter to request checksum information as part of a `HeadObject` request.
+  If specified, the result should contain the checksum for the object if available in the S3 response.
+  ([#1083](https://github.com/awslabs/mountpoint-s3/pull/1083))
+
+### Breaking changes
+
+* `HeadObjectResult` no longer contains an `ObjectInfo` struct.
+  Instead, it returns the object attributes as individual fields on the `HeadObjectResult`.
+  The entity tag field has also changed and is now of type `ETag` rather than `String`.
+  ([#1058](https://github.com/awslabs/mountpoint-s3/pull/1058))
+* `HeadObjectResult` no longer provides the bucket and key used in the original request.
+  ([#1058](https://github.com/awslabs/mountpoint-s3/pull/1058))
+* `head_object` method now requires a `HeadObjectParams` parameter.
+  The structure itself is not required to specify anything to achieve the existing behavior.
+  ([#1083](https://github.com/awslabs/mountpoint-s3/pull/1083))
+
 ## v0.11.0 (October 17, 2024)
 
 ### Breaking changes

mountpoint-s3-client/Cargo.toml

@@ -11,57 +11,57 @@ description = "High-performance Amazon S3 client for Mountpoint for Amazon S3."
 mountpoint-s3-crt = { path = "../mountpoint-s3-crt", version = "0.10.0" }
 mountpoint-s3-crt-sys = { path = "../mountpoint-s3-crt-sys", version = "0.10.0" }
 
-async-trait = "0.1.57"
-auto_impl = "1.1.2"
+async-trait = "0.1.83"
+auto_impl = "1.2.0"
 base64ct = { version = "1.6.0", features = ["std"] }
-const_format = "0.2.30"
-futures = "0.3.24"
-lazy_static = "1.4.0"
-libc = "0.2.126"
-metrics = "0.22.1"
-once_cell = "1.16.0"
-percent-encoding = "2.2.0"
-pin-project = "1.0.12"
-platform-info = "2.0.2"
-regex = "1.7.1"
-serde_json = "1.0.104"
+const_format = "0.2.33"
+futures = "0.3.31"
+lazy_static = "1.5.0"
+libc = "0.2.161"
+metrics = "0.24.0"
+once_cell = "1.20.2"
+percent-encoding = "2.3.1"
+pin-project = "1.1.6"
+platform-info = "2.0.4"
+regex = "1.11.0"
+serde_json = "1.0.132"
 static_assertions = "1.1.0"
-thiserror = "1.0.34"
-time = { version = "0.3.17", features = ["formatting", "parsing"] }
-tracing = { version = "0.1.35", default-features = false, features = ["std", "log"] }
-xmltree = "0.10.3"
+thiserror = "1.0.65"
+time = { version = "0.3.36", features = ["formatting", "parsing"] }
+tracing = { version = "0.1.40", default-features = false, features = ["std", "log"] }
+xmltree = "0.11.0"
 
 # Dependencies for the mock client only
-async-io = { version = "2.3.1", optional = true }
-async-lock = { version = "3.3.0", optional = true }
-md-5 = { version = "0.10.5", optional = true }
+async-io = { version = "2.3.4", optional = true }
+async-lock = { version = "3.4.0", optional = true }
+md-5 = { version = "0.10.6", optional = true }
 rand = { version = "0.8.5", optional = true }
 rand_chacha = { version = "0.3.1", optional = true }
 
 [dev-dependencies]
-anyhow = { version = "1.0.64", features = ["backtrace"] }
-aws-config = "1.2.0"
-aws-credential-types = "1.2.0"
-aws-sdk-s3 = "1.23.0"
-aws-sdk-sts = "1.20.0"
-aws-smithy-runtime-api = "1.2.0"
-bytes = "1.2.1"
-clap = { version = "4.1.9", features = ["derive"] }
-ctor = "0.2.6"
-proptest = "1.4.0"
+anyhow = { version = "1.0.91", features = ["backtrace"] }
+aws-config = "1.5.8"
+aws-credential-types = "1.2.1"
+aws-sdk-s3 = "1.57.0"
+aws-sdk-sts = "1.46.0"
+aws-smithy-runtime-api = "1.7.2"
+bytes = "1.8.0"
+clap = { version = "4.5.20", features = ["derive"] }
+ctor = "0.2.8"
+proptest = "1.5.0"
 rusty-fork = "0.3.0"
-tempfile = "3.5.0"
-test-case = "2.2.2"
-tokio = { version = "1.24.2", features = ["rt", "macros"] }
-tracing-subscriber = { version = "0.3.14", features = ["fmt", "env-filter"] }
+tempfile = "3.13.0"
+test-case = "3.3.1"
+tokio = { version = "1.41.0", features = ["rt", "macros"] }
+tracing-subscriber = { version = "0.3.18", features = ["fmt", "env-filter"] }
 
 # HACK: we want our own tests to use the mock client, but don't want to enable it for consumers by
 # default, so we take a dev-dependency on ourself with that feature enabled.
 # https://github.com/rust-lang/cargo/issues/2911#issuecomment-749580481
 mountpoint-s3-client = { path = ".", features = ["mock"] }
 
 [build-dependencies]
-built = { version = "0.7.1", features = ["git2"] }
+built = { version = "0.7.5", features = ["git2"] }
 
 [features]
 mock = ["dep:async-io", "dep:async-lock", "dep:md-5", "dep:rand", "dep:rand_chacha"]

mountpoint-s3-client/src/failure_client.rs

@@ -17,9 +17,9 @@ use pin_project::pin_project;
 use crate::object_client::{
     CopyObjectError, CopyObjectParams, CopyObjectResult, DeleteObjectError, DeleteObjectResult, ETag, GetBodyPart,
     GetObjectAttributesError, GetObjectAttributesResult, GetObjectError, GetObjectRequest, HeadObjectError,
-    HeadObjectResult, ListObjectsError, ListObjectsResult, ObjectAttribute, ObjectClient, ObjectClientError,
-    ObjectClientResult, PutObjectError, PutObjectParams, PutObjectRequest, PutObjectResult, PutObjectSingleParams,
-    UploadReview,
+    HeadObjectParams, HeadObjectResult, ListObjectsError, ListObjectsResult, ObjectAttribute, ObjectClient,
+    ObjectClientError, ObjectClientResult, PutObjectError, PutObjectParams, PutObjectRequest, PutObjectResult,
+    PutObjectSingleParams, UploadReview,
 };
 
 // Wrapper for injecting failures into a get stream or a put request
@@ -167,9 +167,10 @@ where
         &self,
         bucket: &str,
         key: &str,
+        params: &HeadObjectParams,
     ) -> ObjectClientResult<HeadObjectResult, HeadObjectError, Self::ClientError> {
         (self.head_object_cb)(&mut *self.state.lock().unwrap(), bucket, key)?;
-        self.client.head_object(bucket, key).await
+        self.client.head_object(bucket, key, params).await
     }
 
     async fn put_object(

mountpoint-s3-client/src/lib.rs

@@ -74,11 +74,11 @@ pub mod config {
 /// Types used by all object clients
 pub mod types {
     pub use super::object_client::{
-        Checksum, ChecksumAlgorithm, CopyObjectParams, CopyObjectResult, DeleteObjectResult, ETag, GetBodyPart,
-        GetObjectAttributesParts, GetObjectAttributesResult, GetObjectRequest, HeadObjectResult, ListObjectsResult,
-        ObjectAttribute, ObjectClientResult, ObjectInfo, ObjectPart, PutObjectParams, PutObjectResult,
-        PutObjectSingleParams, PutObjectTrailingChecksums, RestoreStatus, UploadChecksum, UploadReview,
-        UploadReviewPart,
+        Checksum, ChecksumAlgorithm, ChecksumMode, CopyObjectParams, CopyObjectResult, DeleteObjectResult, ETag,
+        GetBodyPart, GetObjectAttributesParts, GetObjectAttributesResult, GetObjectRequest, HeadObjectParams,
+        HeadObjectResult, ListObjectsResult, ObjectAttribute, ObjectClientResult, ObjectInfo, ObjectPart,
+        PutObjectParams, PutObjectResult, PutObjectSingleParams, PutObjectTrailingChecksums, RestoreStatus,
+        UploadChecksum, UploadReview, UploadReviewPart,
     };
 }
 

mountpoint-s3-client/src/mock_client.rs

@@ -25,11 +25,11 @@ use tracing::trace;
 use crate::checksums::crc32c_to_base64;
 use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata};
 use crate::object_client::{
-    Checksum, ChecksumAlgorithm, CopyObjectError, CopyObjectParams, CopyObjectResult, DeleteObjectError,
+    Checksum, ChecksumAlgorithm, ChecksumMode, CopyObjectError, CopyObjectParams, CopyObjectResult, DeleteObjectError,
     DeleteObjectResult, ETag, GetBodyPart, GetObjectAttributesError, GetObjectAttributesParts,
-    GetObjectAttributesResult, GetObjectError, GetObjectRequest, HeadObjectError, HeadObjectResult, ListObjectsError,
-    ListObjectsResult, ObjectAttribute, ObjectClient, ObjectClientError, ObjectClientResult, ObjectInfo, ObjectPart,
-    PutObjectError, PutObjectParams, PutObjectRequest, PutObjectResult, PutObjectSingleParams,
+    GetObjectAttributesResult, GetObjectError, GetObjectRequest, HeadObjectError, HeadObjectParams, HeadObjectResult,
+    ListObjectsError, ListObjectsResult, ObjectAttribute, ObjectClient, ObjectClientError, ObjectClientResult,
+    ObjectInfo, ObjectPart, PutObjectError, PutObjectParams, PutObjectRequest, PutObjectResult, PutObjectSingleParams,
     PutObjectTrailingChecksums, RestoreStatus, UploadReview, UploadReviewPart,
 };
 
@@ -376,6 +376,10 @@ pub struct MockObject {
     etag: ETag,
     parts: Option<MockObjectParts>,
     object_metadata: HashMap<String, String>,
+    /// S3 checksums associated with the object.
+    ///
+    /// Typically, at most one of the checksums should be set.
+    checksum: Checksum,
 }
 
 impl MockObject {
@@ -395,6 +399,7 @@ impl MockObject {
             etag,
             parts: None,
             object_metadata: HashMap::new(),
+            checksum: Checksum::empty(),
         }
     }
 
@@ -408,6 +413,7 @@ impl MockObject {
             etag,
             parts: None,
             object_metadata: HashMap::new(),
+            checksum: Checksum::empty(),
         }
     }
 
@@ -431,6 +437,7 @@ impl MockObject {
             etag,
             parts: None,
             object_metadata: HashMap::new(),
+            checksum: Checksum::empty(),
         }
     }
 
@@ -450,6 +457,10 @@ impl MockObject {
         self.restore_status = restore_status;
     }
 
+    pub fn set_checksum(&mut self, checksum: Checksum) {
+        self.checksum = checksum;
+    }
+
     pub fn len(&self) -> usize {
         self.size
     }
@@ -676,6 +687,7 @@ impl ObjectClient for MockClient {
         &self,
         bucket: &str,
         key: &str,
+        params: &HeadObjectParams,
     ) -> ObjectClientResult<HeadObjectResult, HeadObjectError, Self::ClientError> {
         trace!(bucket, key, "HeadObject");
         self.inc_op_count(Operation::HeadObject);
@@ -686,16 +698,19 @@ impl ObjectClient for MockClient {
 
         let objects = self.objects.read().unwrap();
         if let Some(object) = objects.get(key) {
+            // Checksum information is opt-in
+            let checksum = match params.checksum_mode {
+                Some(ChecksumMode::Enabled) => object.checksum.clone(),
+                None => Checksum::empty(),
+            };
+
             Ok(HeadObjectResult {
-                bucket: bucket.to_string(),
-                object: ObjectInfo {
-                    key: key.to_string(),
-                    size: object.size as u64,
-                    last_modified: object.last_modified,
-                    etag: object.etag.as_str().to_string(),
-                    storage_class: object.storage_class.clone(),
-                    restore_status: object.restore_status,
-                },
+                size: object.size as u64,
+                last_modified: object.last_modified,
+                etag: object.etag.clone(),
+                storage_class: object.storage_class.clone(),
+                restore_status: object.restore_status,
+                checksum,
             })
         } else {
             Err(ObjectClientError::ServiceError(HeadObjectError::NotFound))
@@ -1680,8 +1695,8 @@ mod tests {
         put_request.complete().await.unwrap();
 
         // head_object returns storage class
-        let head_result = client.head_object(bucket, key).await.unwrap();
-        assert_eq!(head_result.object.storage_class.as_deref(), storage_class);
+        let head_result = client.head_object(bucket, key, &HeadObjectParams::new()).await.unwrap();
+        assert_eq!(head_result.storage_class.as_deref(), storage_class);
 
         // list_objects returns storage class
         let list_result = client.list_objects(bucket, None, "/", 1, "").await.unwrap();
@@ -1703,14 +1718,14 @@ mod tests {
         let head_counter_1 = client.new_counter(Operation::HeadObject);
         let delete_counter_1 = client.new_counter(Operation::DeleteObject);
 
-        let _result = client.head_object(bucket, "key").await;
+        let _result = client.head_object(bucket, "key", &HeadObjectParams::new()).await;
         assert_eq!(1, head_counter_1.count());
         assert_eq!(0, delete_counter_1.count());
 
         let head_counter_2 = client.new_counter(Operation::HeadObject);
         assert_eq!(0, head_counter_2.count());
 
-        let _result = client.head_object(bucket, "key").await;
+        let _result = client.head_object(bucket, "key", &HeadObjectParams::new()).await;
         let _result = client.delete_object(bucket, "key").await;
         let _result = client.delete_object(bucket, "key").await;
         let _result = client.delete_object(bucket, "key").await;

mountpoint-s3-client/src/mock_client/throughput_client.rs

@@ -16,8 +16,8 @@ use crate::mock_client::{
 use crate::object_client::{
     CopyObjectError, CopyObjectParams, CopyObjectResult, DeleteObjectError, DeleteObjectResult, ETag, GetBodyPart,
     GetObjectAttributesError, GetObjectAttributesResult, GetObjectError, GetObjectRequest, HeadObjectError,
-    HeadObjectResult, ListObjectsError, ListObjectsResult, ObjectAttribute, ObjectClient, ObjectClientResult,
-    PutObjectError, PutObjectParams, PutObjectResult, PutObjectSingleParams,
+    HeadObjectParams, HeadObjectResult, ListObjectsError, ListObjectsResult, ObjectAttribute, ObjectClient,
+    ObjectClientResult, PutObjectError, PutObjectParams, PutObjectResult, PutObjectSingleParams,
 };
 
 /// A [MockClient] that rate limits overall download throughput to simulate a target network
@@ -168,8 +168,9 @@ impl ObjectClient for ThroughputMockClient {
         &self,
         bucket: &str,
         key: &str,
+        params: &HeadObjectParams,
     ) -> ObjectClientResult<HeadObjectResult, HeadObjectError, Self::ClientError> {
-        self.inner.head_object(bucket, key).await
+        self.inner.head_object(bucket, key, params).await
     }
 
     async fn put_object(