feat(supergroup): retry snuba batch query in supergroups lightweight backfill

src/sentry/tasks/seer/backfill_supergroups_lightweight.py

@@ -22,10 +22,13 @@
 from sentry.taskworker.namespaces import seer_tasks
 from sentry.types.group import UNRESOLVED_SUBSTATUS_CHOICES
 from sentry.utils import metrics
-from sentry.utils.snuba import bulk_snuba_queries
+from sentry.utils.retries import ConditionalRetryPolicy, exponential_delay
+from sentry.utils.snuba import SnubaError, bulk_snuba_queries
 
 logger = logging.getLogger(__name__)
 
+SNUBA_QUERY_MAX_ATTEMPTS = 3
+
 
 @instrumented_task(
     name="sentry.tasks.seer.backfill_supergroups_lightweight.backfill_supergroups_lightweight_for_org",
@@ -280,8 +283,16 @@ def _batch_fetch_events(groups: Sequence[Group], organization_id: int) -> list[t
             )
         )
 
-    results = bulk_snuba_queries(
-        snuba_requests, referrer=Referrer.SUPERGROUPS_BACKFILL_LIGHTWEIGHT_GET_LATEST_EVENTS.value
+    retry_policy = ConditionalRetryPolicy(
+        test_function=lambda attempt, exc: attempt < SNUBA_QUERY_MAX_ATTEMPTS
+        and isinstance(exc, SnubaError),
+        delay_function=exponential_delay(1.0),
+    )
+    results = retry_policy(
+        lambda: bulk_snuba_queries(
+            snuba_requests,
+            referrer=Referrer.SUPERGROUPS_BACKFILL_LIGHTWEIGHT_GET_LATEST_EVENTS.value,
+        )
     )
 
     # Build unfetched Event objects from Snuba results, keeping groups aligned

tests/sentry/tasks/seer/test_backfill_supergroups_lightweight.py

@@ -1,15 +1,18 @@
 from datetime import UTC, datetime, timedelta
 from unittest.mock import MagicMock, patch
 
+import pytest
+
 from sentry.models.group import DEFAULT_TYPE_ID
 from sentry.tasks.seer.backfill_supergroups_lightweight import (
     backfill_supergroups_lightweight_for_org,
 )
 from sentry.testutils.cases import TestCase
-
-TEST_BATCH_SIZE = 5
 from sentry.testutils.helpers.features import with_feature
 from sentry.types.group import GroupSubStatus
+from sentry.utils.snuba import SnubaError
+
+TEST_BATCH_SIZE = 5
 
 
 class BackfillSupergroupsLightweightForOrgTest(TestCase):
@@ -184,6 +187,39 @@ def test_skips_old_groups_with_no_events(self, mock_snuba, mock_request):
 
         mock_request.assert_not_called()
 
+    @with_feature("organizations:supergroups-lightweight-rca-clustering-write")
+    @patch("sentry.utils.retries.time.sleep")
+    @patch(
+        "sentry.tasks.seer.backfill_supergroups_lightweight.make_lightweight_rca_cluster_request"
+    )
+    @patch("sentry.tasks.seer.backfill_supergroups_lightweight.bulk_snuba_queries")
+    def test_retries_snuba_query_on_failure(self, mock_snuba, mock_request, mock_sleep):
+        mock_request.return_value = MagicMock(status=200)
+        mock_snuba.side_effect = [
+            SnubaError("transient"),
+            SnubaError("transient"),
+            [{"data": [{"event_id": self.event.event_id}]}],
+        ]
+
+        backfill_supergroups_lightweight_for_org(self.organization.id)
+
+        assert mock_snuba.call_count == 3
+
+    @with_feature("organizations:supergroups-lightweight-rca-clustering-write")
+    @patch("sentry.utils.retries.time.sleep")
+    @patch(
+        "sentry.tasks.seer.backfill_supergroups_lightweight.make_lightweight_rca_cluster_request"
+    )
+    @patch("sentry.tasks.seer.backfill_supergroups_lightweight.bulk_snuba_queries")
+    def test_raises_after_snuba_retries_exhausted(self, mock_snuba, mock_request, mock_sleep):
+        mock_snuba.side_effect = SnubaError("persistent")
+
+        with pytest.raises(SnubaError):
+            backfill_supergroups_lightweight_for_org(self.organization.id)
+
+        assert mock_snuba.call_count == 3
+        mock_request.assert_not_called()
+
     @with_feature("organizations:supergroups-lightweight-rca-clustering-write")
     @patch(
         "sentry.tasks.seer.backfill_supergroups_lightweight.make_lightweight_rca_cluster_request"