Skip to content

Commit 17d6c53

Browse files
feat(google-api-core): add support for resumable uploads (#18352)
Towards b/457416314, b/556259599 --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
1 parent 5908b74 commit 17d6c53

12 files changed

Lines changed: 8011 additions & 3 deletions

‎packages/google-api-core/google/api_core/exceptions.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -446,6 +446,41 @@ class AsyncRestUnsupportedParameterError(NotImplementedError):
446446
pass
447447

448448

449+
class ResumableTransferError(GoogleAPICallError):
450+
"""Base class for resumable transfer errors."""
451+
452+
upload_url: Optional[str] = None
453+
chunk_size: Optional[int] = None
454+
455+
def __init__(
456+
self,
457+
message: str,
458+
*args,
459+
upload_url: Optional[str] = None,
460+
chunk_size: Optional[int] = None,
461+
**kwargs,
462+
) -> None:
463+
super().__init__(message, *args, **kwargs)
464+
self.upload_url = upload_url
465+
self.chunk_size = chunk_size
466+
467+
468+
class TransferStalledError(ResumableTransferError):
469+
"""Raised when upload throughput stays below minimum rate past stall timeout."""
470+
471+
472+
class UnseekableStreamError(ResumableTransferError):
473+
"""Raised when server recovery requires rewinding a non-seekable stream."""
474+
475+
476+
class UploadCancelledError(ResumableTransferError):
477+
"""Raised when the upload is cancelled by the client or server."""
478+
479+
480+
class MissingStatusHeaderError(ResumableTransferError):
481+
"""Raised when server response lacks the required X-Goog-Upload-Status header."""
482+
483+
449484
def exception_class_for_http_status(status_code):
450485
"""Return the exception class for a specific HTTP status code.
451486
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Resumable transfer library for Google APIs."""
16+
17+
from google.api_core.exceptions import (
18+
MissingStatusHeaderError,
19+
ResumableTransferError,
20+
TransferStalledError,
21+
UnseekableStreamError,
22+
UploadCancelledError,
23+
)
24+
from google.api_core.resumable_transfer.common import (
25+
DEFAULT_CHUNK_SIZE,
26+
ProgressState,
27+
ResumableUploadConfig,
28+
UploadProgress,
29+
)
30+
from google.api_core.resumable_transfer.upload import ResumableUploadSession
31+
from google.api_core.resumable_transfer.upload_async import (
32+
AsyncResumableUploadSession,
33+
AsyncUploadOperation,
34+
)
35+
36+
__all__ = [
37+
"DEFAULT_CHUNK_SIZE",
38+
"MissingStatusHeaderError",
39+
"ProgressState",
40+
"ResumableTransferError",
41+
"TransferStalledError",
42+
"UnseekableStreamError",
43+
"UploadCancelledError",
44+
"UploadProgress",
45+
"ResumableUploadConfig",
46+
"ResumableUploadSession",
47+
"AsyncResumableUploadSession",
48+
"AsyncUploadOperation",
49+
]
Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Common constants, headers, and shared configuration for Resumable Upload protocol."""
16+
17+
import dataclasses
18+
import datetime
19+
import enum
20+
from typing import Any, Mapping, Optional, Sequence, Tuple, Union
21+
22+
import google.protobuf.message
23+
import proto
24+
from google.protobuf import json_format
25+
26+
from google.api_core import exceptions
27+
28+
# Default chunk size: 10 MiB
29+
DEFAULT_CHUNK_SIZE = 10 * 1024 * 1024
30+
31+
# Default timeout in seconds for the initial start request
32+
DEFAULT_START_TIMEOUT = 60.0
33+
34+
# Protocol Headers
35+
HEADER_PROTOCOL = "X-Goog-Upload-Protocol"
36+
HEADER_COMMAND = "X-Goog-Upload-Command"
37+
HEADER_STATUS = "X-Goog-Upload-Status"
38+
HEADER_URL = "X-Goog-Upload-URL"
39+
HEADER_OFFSET = "X-Goog-Upload-Offset"
40+
HEADER_SIZE_RECEIVED = "X-Goog-Upload-Size-Received"
41+
HEADER_CONTENT_TYPE = "X-Goog-Upload-Header-Content-Type"
42+
HEADER_CONTENT_LENGTH = "X-Goog-Upload-Header-Content-Length"
43+
HEADER_CHUNK_GRANULARITY = "X-Goog-Upload-Chunk-Granularity"
44+
45+
PROTOCOL_RESUMABLE = "resumable"
46+
47+
48+
class _Command(str, enum.Enum):
49+
"""Protocol commands."""
50+
51+
START = "start"
52+
UPLOAD = "upload"
53+
FINALIZE = "finalize"
54+
QUERY = "query"
55+
CANCEL = "cancel"
56+
57+
58+
class _Status(str, enum.Enum):
59+
"""Server upload status values."""
60+
61+
ACTIVE = "active"
62+
FINAL = "final"
63+
CANCELLED = "cancelled"
64+
65+
66+
class ProgressState(str, enum.Enum):
67+
"""Progress notification state values."""
68+
69+
STARTED = "started"
70+
UPLOADING = "uploading"
71+
RECOVERING = "recovering"
72+
OFFSET_RECEIVED = "offset received"
73+
FINALIZED = "finalized"
74+
75+
76+
@dataclasses.dataclass(frozen=True)
77+
class UploadProgress:
78+
"""Upload progress notification payload.
79+
80+
Attributes:
81+
upload_url: The unique session URL for this upload.
82+
chunk_size: The actual negotiated chunk size.
83+
bytes_uploaded: The total confirmed bytes committed so far.
84+
total_bytes: The total size of the stream in bytes, if known.
85+
state: The current progress state.
86+
"""
87+
88+
upload_url: str
89+
chunk_size: int
90+
bytes_uploaded: int
91+
total_bytes: Optional[int]
92+
state: ProgressState
93+
94+
95+
# HTTP status codes indicating transient retryable errors (Category 1)
96+
RETRYABLE_STATUS_CODES = (408, 429, 500, 502, 503, 504)
97+
98+
# HTTP status codes indicating state consistency errors requiring recovery (Category 2)
99+
RECOVERABLE_STATUS_CODES = (400, 412, 416)
100+
101+
# Exception types indicating unrecoverable terminal conditions (Category 3)
102+
TERMINAL_ERRORS = (
103+
exceptions.DeadlineExceeded,
104+
exceptions.TransferStalledError,
105+
exceptions.UploadCancelledError,
106+
exceptions.UnseekableStreamError,
107+
)
108+
109+
110+
@dataclasses.dataclass
111+
class ResumableUploadConfig:
112+
"""Configuration options for a resumable upload.
113+
114+
Attributes:
115+
chunk_size: Size in bytes for each uploaded data chunk. Defaults to 10 MiB.
116+
stall_minimum_rate: Minimum transfer rate in bytes per second. Defaults to 64 KiB/s.
117+
stall_timeout: Stall duration threshold in seconds. Defaults to 120s.
118+
headers: Additional HTTP headers dispatched exclusively with start request.
119+
deadline: Optional overall wall-clock deadline for the entire upload process.
120+
When set, each HTTP request timeout is trimmed to the remaining time before
121+
the deadline, and DeadlineExceeded is raised immediately when the deadline
122+
elapses (even on healthy streams). Timezone-aware datetimes (e.g.,
123+
``datetime.now(timezone.utc) + timedelta(...)``) are recommended;
124+
timezone-naive datetimes are assumed to be in local system time. When
125+
None (default), transfer duration is governed by stall control
126+
(stall_minimum_rate and stall_timeout), allowing healthy streams
127+
transferring above the minimum rate to continue indefinitely.
128+
"""
129+
130+
chunk_size: int = DEFAULT_CHUNK_SIZE
131+
stall_minimum_rate: int = 64 * 1024
132+
stall_timeout: float = 120.0
133+
headers: Optional[Union[Mapping[str, str], Sequence[Tuple[str, str]]]] = None
134+
deadline: Optional[datetime.datetime] = None
135+
136+
def __post_init__(self) -> None:
137+
if self.deadline is not None:
138+
self.deadline = self.deadline.astimezone(datetime.timezone.utc)
139+
140+
@property
141+
def start_headers(self) -> Optional[Sequence[Tuple[str, str]]]:
142+
"""Returns normalized additional headers for the start request."""
143+
if self.headers is None:
144+
return None
145+
if isinstance(self.headers, Mapping):
146+
return list(self.headers.items())
147+
return list(self.headers)
148+
149+
150+
def _format_response_payload(
151+
response: Union[Any, bytes],
152+
response_type: Optional[Any],
153+
) -> Any:
154+
"""Formats raw response or bytes into protobuf or proto-plus message type if configured.
155+
156+
Args:
157+
response: Raw HTTP response object or response body bytes.
158+
response_type: Deserializer callable, proto.Message class, or
159+
google.protobuf.message.Message class or instance.
160+
161+
Returns:
162+
Deserialized protobuf message, or raw response bytes if
163+
``response_type`` is ``None``.
164+
"""
165+
content: bytes
166+
if isinstance(response, bytes):
167+
content = response
168+
elif hasattr(response, "content"):
169+
content = response.content
170+
else:
171+
content = bytes(response)
172+
173+
if response_type is None:
174+
return content
175+
176+
from_json_fn = getattr(response_type, "from_json", None)
177+
if callable(from_json_fn):
178+
if isinstance(response_type, type) and issubclass(response_type, proto.Message):
179+
return from_json_fn(content, ignore_unknown_fields=True)
180+
return from_json_fn(content)
181+
if isinstance(response_type, type) and issubclass(
182+
response_type, google.protobuf.message.Message
183+
):
184+
instance = response_type()
185+
return json_format.Parse(content, instance, ignore_unknown_fields=True)
186+
if isinstance(response_type, google.protobuf.message.Message):
187+
return json_format.Parse(content, response_type, ignore_unknown_fields=True)
188+
if callable(response_type):
189+
return response_type(content)
190+
191+
return content

0 commit comments

Comments
 (0)