Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/storage/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ pub enum ReadError {
ShortRead(u64),

/// The read received more bytes than expected.
#[error("too many bytes received: expected {expected}, stopped download at {got}")]
#[error("too many bytes received: expected {expected}, stopped read at {got}")]
LongRead { got: u64, expected: u64 },

/// Only 200 and 206 status codes are expected in successful responses.
Expand Down
2 changes: 1 addition & 1 deletion src/storage/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ pub use gax::Result;
pub use gax::error::Error;

pub mod backoff_policy;
pub mod download_resume_policy;
pub mod read_resume_policy;
pub mod retry_policy;
pub use crate::storage::checksum;
pub use crate::storage::upload_source;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.

//! Defines the download resume policies for Google Cloud Storage.
//! Defines the read resume policies for Google Cloud Storage.
//!
//! Even if a download request starts successfully, it may be fail after it
//! starts. For example, the download may be interrupted or become too slow and
//! "stall". The client library can automatically recover from such errors. The
//! application may want to control what errors are treated as recoverable, and
//! how many failures are tolerated before abandoning the download.
//! Even if a read request starts successfully, it may be fail after it starts.
//! For example, the read may be interrupted or become too slow and "stall". The
//! client library can automatically recover from such errors. The application
//! may want to control what errors are treated as recoverable, and how many
//! failures are tolerated before abandoning the read request.
//!
//! The traits and types defined in this module allow for such customization.
//!
//! # Example
//! ```
//! # use google_cloud_storage::download_resume_policy::*;
//! # use google_cloud_storage::read_resume_policy::*;
//! let policy = Recommended.with_attempt_limit(3);
//! assert!(matches!(policy.on_error(&ResumeQuery::new(0), io_error()), ResumeResult::Continue(_)));
//! assert!(matches!(policy.on_error(&ResumeQuery::new(1), io_error()), ResumeResult::Continue(_)));
Expand All @@ -34,7 +34,7 @@
//! use gax::error::{Error, rpc::Code, rpc::Status};
//! fn io_error() -> Error {
//! // ... details omitted ...
//! # Error::io("something failed in the download")
//! # Error::io("something failed in the read request")
//! }
//! ```

Expand All @@ -43,14 +43,14 @@ use crate::Error;
pub use gax::retry_result::RetryResult as ResumeResult;

/// Defines the interface to resume policies.
pub trait DownloadResumePolicy: Send + Sync + std::fmt::Debug {
/// Determines if the download should continue after an error.
pub trait ReadResumePolicy: Send + Sync + std::fmt::Debug {
/// Determines if the read should continue after an error.
fn on_error(&self, status: &ResumeQuery, error: Error) -> ResumeResult;
}

/// Extension trait for [DownloadResumePolicy].
pub trait DownloadResumePolicyExt: Sized {
/// Decorates a [DownloadResumePolicy] to limit the number of resume attempts.
/// Extension trait for [ReadResumePolicy].
pub trait ReadResumePolicyExt: Sized {
/// Decorates a [ReadResumePolicy] to limit the number of resume attempts.
///
/// This policy decorates an inner policy and limits the total number of
/// attempts. Note that `on_error()` is not called before the initial
Expand All @@ -64,7 +64,7 @@ pub trait DownloadResumePolicyExt: Sized {
///
/// # Example
/// ```
/// # use google_cloud_storage::download_resume_policy::*;
/// # use google_cloud_storage::read_resume_policy::*;
/// let policy = Recommended.with_attempt_limit(3);
/// assert!(matches!(policy.on_error(&ResumeQuery::new(0), transient_error()), ResumeResult::Continue(_)));
/// assert!(matches!(policy.on_error(&ResumeQuery::new(1), transient_error()), ResumeResult::Continue(_)));
Expand All @@ -74,27 +74,27 @@ pub trait DownloadResumePolicyExt: Sized {
/// use gax::error::{Error, rpc::Code, rpc::Status};
/// fn transient_error() -> Error {
/// // ... details omitted ...
/// # Error::io("something failed in the download")
/// # Error::io("something failed in the read request")
/// }
/// ```
fn with_attempt_limit(self, maximum_attempts: u32) -> LimitedAttemptCount<Self> {
LimitedAttemptCount::new(self, maximum_attempts)
}
}
impl<T: DownloadResumePolicy> DownloadResumePolicyExt for T {}
impl<T: ReadResumePolicy> ReadResumePolicyExt for T {}

/// The inputs into a resume policy query.
///
/// On an error, the client library queries the resume policy as to whether it
/// should attempt a new download or not. The client library provides
/// an instance of this type to the resume policy.
/// should attempt a new read request or not. The client library provides an
/// instance of this type to the resume policy.
///
/// We use a struct so we can grow the amount of information without breaking
/// existing resume policies.
#[derive(Debug)]
#[non_exhaustive]
pub struct ResumeQuery {
/// The number of times the download has been interrupted already.
/// The number of times the read request has been interrupted already.
pub attempt_count: u32,
}

Expand All @@ -105,32 +105,32 @@ impl ResumeQuery {
}
}

/// The recommended policy for storage downloads.
/// The recommended policy when reading objects from Cloud Storage.
///
/// This policy resumes any download that fails due to I/O errors, and stops on
/// any other error kind.
/// This policy resumes any read that fails due to I/O errors, and stops on any
/// other error kind.
///
/// # Example
/// ```
/// # use google_cloud_storage::download_resume_policy::*;
/// # use google_cloud_storage::read_resume_policy::*;
/// let policy = Recommended;
/// assert!(matches!(policy.on_error(&ResumeQuery::new(0), io_error()), ResumeResult::Continue(_)));
/// assert!(matches!(policy.on_error(&ResumeQuery::new(0), other_error()), ResumeResult::Permanent(_)));
///
/// use gax::error::{Error, rpc::Code, rpc::Status};
/// fn io_error() -> Error {
/// // ... details omitted ...
/// # Error::io("something failed in the download")
/// # Error::io("something failed in the read request")
/// }
/// fn other_error() -> Error {
/// // ... details omitted ...
/// # Error::deser("something failed in the download")
/// # Error::deser("something failed in the read request")
/// }
/// ```
#[derive(Debug)]
pub struct Recommended;

impl DownloadResumePolicy for Recommended {
impl ReadResumePolicy for Recommended {
fn on_error(&self, _status: &ResumeQuery, error: Error) -> ResumeResult {
if error.is_io() {
ResumeResult::Continue(error)
Expand All @@ -147,7 +147,7 @@ impl DownloadResumePolicy for Recommended {
///
/// # Example
/// ```
/// # use google_cloud_storage::download_resume_policy::*;
/// # use google_cloud_storage::read_resume_policy::*;
/// let policy = AlwaysResume.with_attempt_limit(3);
/// assert!(matches!(policy.on_error(&ResumeQuery::new(0), scary_error()), ResumeResult::Continue(_)));
/// assert!(matches!(policy.on_error(&ResumeQuery::new(1), scary_error()), ResumeResult::Continue(_)));
Expand All @@ -157,13 +157,13 @@ impl DownloadResumePolicy for Recommended {
/// use gax::error::{Error, rpc::Code, rpc::Status};
/// fn scary_error() -> Error {
/// // ... details omitted ...
/// # Error::deser("something failed in the download")
/// # Error::deser("something failed in the read request")
/// }
/// ```
#[derive(Debug)]
pub struct AlwaysResume;

impl DownloadResumePolicy for AlwaysResume {
impl ReadResumePolicy for AlwaysResume {
fn on_error(&self, _status: &ResumeQuery, error: Error) -> ResumeResult {
ResumeResult::Continue(error)
}
Expand All @@ -175,7 +175,7 @@ impl DownloadResumePolicy for AlwaysResume {
///
/// # Example
/// ```
/// # use google_cloud_storage::download_resume_policy::*;
/// # use google_cloud_storage::read_resume_policy::*;
/// let policy = NeverResume.with_attempt_limit(3);
/// assert!(matches!(policy.on_error(&ResumeQuery::new(0), io_error()), ResumeResult::Permanent(_)));
/// assert!(matches!(policy.on_error(&ResumeQuery::new(1), io_error()), ResumeResult::Permanent(_)));
Expand All @@ -185,12 +185,12 @@ impl DownloadResumePolicy for AlwaysResume {
/// use gax::error::{Error, rpc::Code, rpc::Status};
/// fn io_error() -> Error {
/// // ... details omitted ...
/// # Error::io("something failed in the download")
/// # Error::io("something failed in the read request")
/// }
/// ```
#[derive(Debug)]
pub struct NeverResume;
impl DownloadResumePolicy for NeverResume {
impl ReadResumePolicy for NeverResume {
fn on_error(&self, _status: &ResumeQuery, error: Error) -> ResumeResult {
ResumeResult::Permanent(error)
}
Expand All @@ -200,7 +200,7 @@ impl DownloadResumePolicy for NeverResume {
///
/// # Example
/// ```
/// # use google_cloud_storage::download_resume_policy::*;
/// # use google_cloud_storage::read_resume_policy::*;
/// let policy = LimitedAttemptCount::new(AlwaysResume, 3);
/// assert!(matches!(policy.on_error(&ResumeQuery::new(0), scary_error()), ResumeResult::Continue(_)));
/// assert!(matches!(policy.on_error(&ResumeQuery::new(1), scary_error()), ResumeResult::Continue(_)));
Expand All @@ -210,7 +210,7 @@ impl DownloadResumePolicy for NeverResume {
/// use gax::error::{Error, rpc::Code, rpc::Status};
/// fn scary_error() -> Error {
/// // ... details omitted ...
/// # Error::deser("something failed in the download")
/// # Error::deser("something failed in the read request")
/// }
/// ```
#[derive(Debug)]
Expand All @@ -229,9 +229,9 @@ impl<P> LimitedAttemptCount<P> {
}
}

impl<P> DownloadResumePolicy for LimitedAttemptCount<P>
impl<P> ReadResumePolicy for LimitedAttemptCount<P>
where
P: DownloadResumePolicy,
P: ReadResumePolicy,
{
fn on_error(&self, status: &ResumeQuery, error: Error) -> ResumeResult {
match self.inner.on_error(status, error) {
Expand Down
2 changes: 1 addition & 1 deletion src/storage/src/storage/checksum.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ use crate::model::ObjectChecksums;
/// Computes a checksum or hash for [Cloud Storage] transfers.
///
/// We want to minimize code complexity in our implementation of data integrity
/// checks for uploads and downloads. This trait defines a composable interface
/// checks for writes and reads. This trait defines a composable interface
/// to support:
/// - No checksums (`Null`): the client library does not compute any checksums,
/// and therefore does not validate checksums either.
Expand Down
26 changes: 13 additions & 13 deletions src/storage/src/storage/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ use super::request_options::RequestOptions;
use crate::Error;
use crate::builder::storage::ReadObject;
use crate::builder::storage::UploadObject;
use crate::download_resume_policy::DownloadResumePolicy;
use crate::error::KeyAes256Error;
use crate::read_resume_policy::ReadResumePolicy;
use crate::storage::checksum::details::Crc32c;
use crate::upload_source::Payload;
use auth::credentials::CacheableResource;
Expand Down Expand Up @@ -173,7 +173,7 @@ impl Storage {
UploadObject::new(self.inner.clone(), bucket, object, payload)
}

/// Downloads the contents of an object.
/// Reads the contents of an object.
///
/// # Example
/// ```
Expand Down Expand Up @@ -513,29 +513,29 @@ impl ClientBuilder {
self
}

/// Configure the resume policy for downloads.
/// Configure the resume policy for object reads.
///
/// The Cloud Storage client library can automatically resume a download
/// The Cloud Storage client library can automatically resume a read request
/// that is interrupted by a transient error. Applications may want to
/// limit the number of download attempts, or may wish to expand the type
/// limit the number of read attempts, or may wish to expand the type
/// of errors treated as retryable.
///
/// # Example
/// ```
/// # use google_cloud_storage::client::Storage;
/// # async fn sample() -> anyhow::Result<()> {
/// use google_cloud_storage::download_resume_policy::{AlwaysResume, DownloadResumePolicyExt};
/// use google_cloud_storage::read_resume_policy::{AlwaysResume, ReadResumePolicyExt};
/// let client = Storage::builder()
/// .with_download_resume_policy(AlwaysResume.with_attempt_limit(3))
/// .with_read_resume_policy(AlwaysResume.with_attempt_limit(3))
/// .build()
/// .await?;
/// # Ok(()) }
/// ```
pub fn with_download_resume_policy<V>(mut self, v: V) -> Self
pub fn with_read_resume_policy<V>(mut self, v: V) -> Self
where
V: DownloadResumePolicy + 'static,
V: ReadResumePolicy + 'static,
{
self.default_options.download_resume_policy = Arc::new(v);
self.default_options.read_resume_policy = Arc::new(v);
self
}
}
Expand Down Expand Up @@ -781,10 +781,10 @@ pub(crate) mod tests {

mockall::mock! {
#[derive(Debug)]
pub DownloadResumePolicy {}
pub ReadResumePolicy {}

impl crate::download_resume_policy::DownloadResumePolicy for DownloadResumePolicy {
fn on_error(&self, query: &crate::download_resume_policy::ResumeQuery, error: gax::error::Error) -> crate::download_resume_policy::ResumeResult;
impl crate::read_resume_policy::ReadResumePolicy for ReadResumePolicy {
fn on_error(&self, query: &crate::read_resume_policy::ResumeQuery, error: gax::error::Error) -> crate::read_resume_policy::ResumeResult;
}
}
}
Loading
Loading