Skip to content

Commit c11ed49

Browse files
committed
Allow '_shards.skipped' to be absent on scroll responses
1 parent 6d30732 commit c11ed49

File tree

3 files changed

+57
-22
lines changed

3 files changed

+57
-22
lines changed

elasticsearch/_async/helpers.py

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -332,24 +332,28 @@ async def async_scan(
332332
for hit in resp["hits"]["hits"]:
333333
yield hit
334334

335+
# Default to 0 if the value isn't included in the response
336+
shards_successful = resp["_shards"].get("successful", 0)
337+
shards_skipped = resp["_shards"].get("skipped", 0)
338+
shards_total = resp["_shards"].get("total", 0)
339+
335340
# check if we have any errors
336-
if (resp["_shards"]["successful"] + resp["_shards"]["skipped"]) < resp[
337-
"_shards"
338-
]["total"]:
341+
if (shards_successful + shards_skipped) < shards_total:
342+
shards_message = "Scroll request has only succeeded on %d (+%d skipped) shards out of %d."
339343
logger.warning(
340-
"Scroll request has only succeeded on %d (+%d skipped) shards out of %d.",
341-
resp["_shards"]["successful"],
342-
resp["_shards"]["skipped"],
343-
resp["_shards"]["total"],
344+
shards_message,
345+
shards_successful,
346+
shards_skipped,
347+
shards_total,
344348
)
345349
if raise_on_error:
346350
raise ScanError(
347351
scroll_id,
348-
"Scroll request has only succeeded on %d (+%d skipped) shards out of %d."
352+
shards_message
349353
% (
350-
resp["_shards"]["successful"],
351-
resp["_shards"]["skipped"],
352-
resp["_shards"]["total"],
354+
shards_successful,
355+
shards_skipped,
356+
shards_total,
353357
),
354358
)
355359
resp = await client.scroll(

elasticsearch/helpers/actions.py

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -531,24 +531,28 @@ def scan(
531531
for hit in resp["hits"]["hits"]:
532532
yield hit
533533

534+
# Default to 0 if the value isn't included in the response
535+
shards_successful = resp["_shards"].get("successful", 0)
536+
shards_skipped = resp["_shards"].get("skipped", 0)
537+
shards_total = resp["_shards"].get("total", 0)
538+
534539
# check if we have any errors
535-
if (resp["_shards"]["successful"] + resp["_shards"]["skipped"]) < resp[
536-
"_shards"
537-
]["total"]:
540+
if (shards_successful + shards_skipped) < shards_total:
541+
shards_message = "Scroll request has only succeeded on %d (+%d skipped) shards out of %d."
538542
logger.warning(
539-
"Scroll request has only succeeded on %d (+%d skipped) shards out of %d.",
540-
resp["_shards"]["successful"],
541-
resp["_shards"]["skipped"],
542-
resp["_shards"]["total"],
543+
shards_message,
544+
shards_successful,
545+
shards_skipped,
546+
shards_total,
543547
)
544548
if raise_on_error:
545549
raise ScanError(
546550
scroll_id,
547-
"Scroll request has only succeeded on %d (+%d skipped) shards out of %d."
551+
shards_message
548552
% (
549-
resp["_shards"]["successful"],
550-
resp["_shards"]["skipped"],
551-
resp["_shards"]["total"],
553+
shards_successful,
554+
shards_skipped,
555+
shards_total,
552556
),
553557
)
554558
resp = client.scroll(

test_elasticsearch/test_server/test_helpers.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,33 @@ def test_clear_scroll(self):
500500
)
501501
spy.assert_not_called()
502502

503+
def test_shards_no_skipped_field(self):
504+
with patch.object(self, "client") as client_mock:
505+
client_mock.search.return_value = {
506+
"_scroll_id": "dummy_id",
507+
"_shards": {"successful": 5, "total": 5},
508+
"hits": {"hits": [{"search_data": 1}]},
509+
}
510+
client_mock.scroll.side_effect = [
511+
{
512+
"_scroll_id": "dummy_id",
513+
"_shards": {"successful": 4, "total": 5},
514+
"hits": {"hits": [{"scroll_data": 42}]},
515+
},
516+
{
517+
"_scroll_id": "dummy_id",
518+
"_shards": {"successful": 4, "total": 5},
519+
"hits": {"hits": []},
520+
},
521+
]
522+
523+
data = list(
524+
helpers.scan(
525+
self.client, index="test_index", size=2, raise_on_error=True
526+
)
527+
)
528+
self.assertEqual(data, [{"search_data": 1}, {"scroll_data": 42}])
529+
503530

504531
class TestReindex(ElasticsearchTestCase):
505532
def setup_method(self, _):

0 commit comments

Comments
 (0)