Skip to content

Commit 3b6e96e

Browse files
authored
fix: Send request queue write fields under the names the API declares (#1026)
`add_request` and `batch_add_requests` sent request fields snake_cased. The API declares its write bodies with `additionalProperties: false`, so it rejected the whole write with HTTP 400 — a request carrying `user_data`, `no_retry`, `retry_count`, `loaded_url`, `error_messages`, or `handled_at` could not be added at all. The identical dict passed to `update_request` worked: ```python add_request -> 400 {"uniqueKey": "k", "url": "...", "user_data": {...}, "no_retry": true} update_request -> 200 {"uniqueKey": "k", "url": "...", "userData": {...}, "noRetry": true} ``` The client was using `RequestDraft` (the spec's response-only schema for `unprocessedRequests`, which declares only `id`/`unique_key`/`url`/`method` and leaves the rest to `extra='allow'`) as its input model. The spec actually declares both add-request bodies as `RequestWithoutId`, now correctly generated as its own model since apify-docs#2774 fixed the spec bundler dropping its identity — no custom postprocessing needed. Upstream spec fix ([apify-docs#2774](apify/apify-docs#2774)) is closed. Two things to know: - **Validation narrows.** The newly declared fields are validated instead of passed through, so `retry_count='abc'` or a naive `handled_at` now raise `ValidationError` instead of a 400 from the API. - **`update_request`'s timestamp format changes** (`fa77d88`): `mode='json'` emits ISO 8601 where python mode emitted `2019-06-16 10:23:31.607000+00:00`. The API accepts both, so this is spec fidelity, not a fix — its own commit, revertible alone. Integration tests round-trip every field through both add paths against the live API; they fail on master with `InvalidRequestError`. *✍️ Drafted by Claude Code*
1 parent f1d5a06 commit 3b6e96e

6 files changed

Lines changed: 518 additions & 88 deletions

File tree

scripts/postprocess_generated_models.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@
5656
RESOURCE_INPUT_TYPEDDICTS: frozenset[str] = frozenset(
5757
{
5858
'Request', # RequestQueueClient.update_request
59-
'RequestDraft', # RequestQueueClient.add_request, batch_add_requests
59+
'RequestWithoutId', # RequestQueueClient.add_request, batch_add_requests
6060
'RequestDraftDelete', # RequestQueueClient.batch_delete_requests
6161
'TaskInput', # Actor/Task start/call/update default input
6262
'WebhookCreate', # Actor/Task start/call webhook list element

src/apify_client/_resource_clients/request_queue.py

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
RequestQueueResponse,
3333
RequestRegistration,
3434
RequestResponse,
35+
RequestWithoutId,
3536
UnlockRequestsResponse,
3637
UnlockRequestsResult,
3738
)
@@ -50,10 +51,10 @@
5051
from apify_client._typeddicts import (
5152
RequestCamelDict,
5253
RequestDict,
53-
RequestDraftCamelDict,
5454
RequestDraftDeleteCamelDict,
5555
RequestDraftDeleteDict,
56-
RequestDraftDict,
56+
RequestWithoutIdCamelDict,
57+
RequestWithoutIdDict,
5758
)
5859
from apify_client.types import Timeout
5960

@@ -68,7 +69,7 @@
6869

6970

7071
def _serialize_requests(
71-
requests: list[RequestDraft] | list[RequestDraftDict] | list[RequestDraftCamelDict],
72+
requests: list[RequestWithoutId] | list[RequestWithoutIdDict] | list[RequestWithoutIdCamelDict],
7273
) -> list[bytes]:
7374
"""Validate requests and serialize each one into the JSON bytes it will occupy in the batch request body.
7475
@@ -80,8 +81,8 @@ def _serialize_requests(
8081
"""
8182
return [
8283
json.dumps(
83-
(request if isinstance(request, RequestDraft) else RequestDraft.model_validate(request)).model_dump(
84-
by_alias=True, exclude_none=True
84+
(request if isinstance(request, RequestWithoutId) else RequestWithoutId.model_validate(request)).model_dump(
85+
mode='json', by_alias=True, exclude_none=True, fallback=str
8586
),
8687
ensure_ascii=False,
8788
allow_nan=False,
@@ -228,7 +229,7 @@ def list_and_lock_head(
228229

229230
def add_request(
230231
self,
231-
request: RequestDraftDict | RequestDraftCamelDict | RequestDraft,
232+
request: RequestWithoutIdDict | RequestWithoutIdCamelDict | RequestWithoutId,
232233
*,
233234
forefront: bool | None = None,
234235
timeout: Timeout = 'short',
@@ -238,22 +239,22 @@ def add_request(
238239
https://docs.apify.com/api/v2#/reference/request-queues/request-collection/add-request
239240
240241
Args:
241-
request: The request to add to the queue.
242+
request: The request to add to the queue. Must carry a `unique_key` and a `url`.
242243
forefront: Whether to add the request to the head or the end of the queue.
243244
timeout: Timeout for the API HTTP request.
244245
245246
Returns:
246247
The added request.
247248
"""
248-
if not isinstance(request, RequestDraft):
249-
request = RequestDraft.model_validate(request)
249+
if not isinstance(request, RequestWithoutId):
250+
request = RequestWithoutId.model_validate(request)
250251

251252
request_params = self._build_params(forefront=forefront, clientKey=self.client_key)
252253

253254
response = self._http_client.call(
254255
url=self._build_url('requests'),
255256
method='POST',
256-
json=request.model_dump(by_alias=True, exclude_none=True),
257+
json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str),
257258
params=request_params,
258259
timeout=timeout,
259260
)
@@ -321,7 +322,7 @@ def update_request(
321322
response = self._http_client.call(
322323
url=self._build_url(f'requests/{to_path_segment(request.id)}'),
323324
method='PUT',
324-
json=request.model_dump(by_alias=True, exclude_none=True),
325+
json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str),
325326
params=request_params,
326327
timeout=timeout,
327328
)
@@ -410,7 +411,7 @@ def delete_request_lock(
410411

411412
def batch_add_requests(
412413
self,
413-
requests: list[RequestDraft] | list[RequestDraftDict] | list[RequestDraftCamelDict],
414+
requests: list[RequestWithoutId] | list[RequestWithoutIdDict] | list[RequestWithoutIdCamelDict],
414415
*,
415416
forefront: bool = False,
416417
max_parallel: int = 1,
@@ -423,7 +424,7 @@ def batch_add_requests(
423424
https://docs.apify.com/api/v2#/reference/request-queues/batch-request-operations/add-requests
424425
425426
Args:
426-
requests: List of requests to be added to the queue.
427+
requests: List of requests to be added to the queue. Each must carry a `unique_key` and a `url`.
427428
forefront: Whether to add requests to the front of the queue.
428429
max_parallel: Specifies the maximum number of parallel tasks for API calls. This is only applicable
429430
to the async client. For the sync client, this value must be set to 1, as parallel execution
@@ -510,7 +511,7 @@ def batch_delete_requests(
510511
else RequestDraftDelete.model_validate(
511512
request,
512513
)
513-
).model_dump(by_alias=True, exclude_none=True)
514+
).root.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str)
514515
for request in requests
515516
]
516517

@@ -761,7 +762,7 @@ async def list_and_lock_head(
761762

762763
async def add_request(
763764
self,
764-
request: RequestDraftDict | RequestDraftCamelDict | RequestDraft,
765+
request: RequestWithoutIdDict | RequestWithoutIdCamelDict | RequestWithoutId,
765766
*,
766767
forefront: bool | None = None,
767768
timeout: Timeout = 'short',
@@ -771,22 +772,22 @@ async def add_request(
771772
https://docs.apify.com/api/v2#/reference/request-queues/request-collection/add-request
772773
773774
Args:
774-
request: The request to add to the queue.
775+
request: The request to add to the queue. Must carry a `unique_key` and a `url`.
775776
forefront: Whether to add the request to the head or the end of the queue.
776777
timeout: Timeout for the API HTTP request.
777778
778779
Returns:
779780
The added request.
780781
"""
781-
if not isinstance(request, RequestDraft):
782-
request = RequestDraft.model_validate(request)
782+
if not isinstance(request, RequestWithoutId):
783+
request = RequestWithoutId.model_validate(request)
783784

784785
request_params = self._build_params(forefront=forefront, clientKey=self.client_key)
785786

786787
response = await self._http_client.call(
787788
url=self._build_url('requests'),
788789
method='POST',
789-
json=request.model_dump(by_alias=True, exclude_none=True),
790+
json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str),
790791
params=request_params,
791792
timeout=timeout,
792793
)
@@ -852,7 +853,7 @@ async def update_request(
852853
response = await self._http_client.call(
853854
url=self._build_url(f'requests/{to_path_segment(request.id)}'),
854855
method='PUT',
855-
json=request.model_dump(by_alias=True, exclude_none=True),
856+
json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str),
856857
params=request_params,
857858
timeout=timeout,
858859
)
@@ -989,7 +990,7 @@ async def _batch_add_requests_worker(
989990

990991
async def batch_add_requests(
991992
self,
992-
requests: list[RequestDraft] | list[RequestDraftDict] | list[RequestDraftCamelDict],
993+
requests: list[RequestWithoutId] | list[RequestWithoutIdDict] | list[RequestWithoutIdCamelDict],
993994
*,
994995
forefront: bool = False,
995996
max_parallel: int = 5,
@@ -1002,7 +1003,7 @@ async def batch_add_requests(
10021003
https://docs.apify.com/api/v2#/reference/request-queues/batch-request-operations/add-requests
10031004
10041005
Args:
1005-
requests: List of requests to be added to the queue.
1006+
requests: List of requests to be added to the queue. Each must carry a `unique_key` and a `url`.
10061007
forefront: Whether to add requests to the front of the queue.
10071008
max_parallel: Specifies the maximum number of parallel tasks for API calls. This is only applicable
10081009
to the async client. For the sync client, this value must be set to 1, as parallel execution
@@ -1094,7 +1095,7 @@ async def batch_delete_requests(
10941095
else RequestDraftDelete.model_validate(
10951096
request,
10961097
)
1097-
).model_dump(by_alias=True, exclude_none=True)
1098+
).root.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str)
10981099
for request in requests
10991100
]
11001101

src/apify_client/_typeddicts.py

Lines changed: 10 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -111,44 +111,6 @@ class RequestCamelDict(RequestBaseCamelDict):
111111
"""
112112

113113

114-
@docs_group('Typed dicts')
115-
class RequestDraftDict(TypedDict):
116-
"""A request that failed to be processed during a request queue operation and can be retried."""
117-
118-
id: NotRequired[str]
119-
"""
120-
A unique identifier assigned to the request.
121-
"""
122-
unique_key: str
123-
"""
124-
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
125-
"""
126-
url: str
127-
"""
128-
The URL of the request.
129-
"""
130-
method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']]
131-
132-
133-
@docs_group('Typed dicts')
134-
class RequestDraftCamelDict(TypedDict):
135-
"""A request that failed to be processed during a request queue operation and can be retried."""
136-
137-
id: NotRequired[str]
138-
"""
139-
A unique identifier assigned to the request.
140-
"""
141-
uniqueKey: str
142-
"""
143-
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
144-
"""
145-
url: str
146-
"""
147-
The URL of the request.
148-
"""
149-
method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']]
150-
151-
152114
@docs_group('Typed dicts')
153115
class RequestDraftDeleteByIdDict(TypedDict):
154116
"""A request that should be deleted, identified by its ID."""
@@ -221,6 +183,16 @@ class RequestDraftDeleteByUniqueKeyCamelDict(TypedDict):
221183
RequestUserDataCamelDict: TypeAlias = dict[str, Any]
222184

223185

186+
@docs_group('Typed dicts')
187+
class RequestWithoutIdDict(RequestBaseDict):
188+
"""A request stored in the request queue, including its metadata and processing state, without the assigned ID."""
189+
190+
191+
@docs_group('Typed dicts')
192+
class RequestWithoutIdCamelDict(RequestBaseCamelDict):
193+
"""A request stored in the request queue, including its metadata and processing state, without the assigned ID."""
194+
195+
224196
TaskInputDict: TypeAlias = dict[str, Any]
225197

226198
TaskInputCamelDict: TypeAlias = dict[str, Any]

0 commit comments

Comments
 (0)