add backoff on Service crd

threescale_api_crd/defaults.py

@@ -1,5 +1,6 @@
 """ Module with default objects """
 
+import backoff
 import logging
 import copy
 import random
@@ -259,21 +260,24 @@ def _set_provider_ref_new_crd(self, spec):
         return spec
 
     def _is_ready(self, obj):
-        """Is object ready?"""
+        """Is object ready?
+
+        Ready states:
+        - Synced=True or Ready=True (with valid ID)
+        - Orphan=True (waiting for parent resource) with no Failed/Invalid
+        """
         if not ("status" in obj.model and "conditions" in obj.model.status):
             return False
         status = obj.as_dict()["status"]
         new_id = status.get(self.ID_NAME, 0)
-        state = {"Failed": True, "Invalid": True, "Synced": False, "Ready": False}
+        state = {"Failed": True, "Invalid": True, "Synced": False, "Ready": False, "Orphan": False}
         for sta in status["conditions"]:
             state[sta["type"]] = sta["status"] == "True"
 
-        return (
-            not state["Failed"]
-            and not state["Invalid"]
-            and (state["Synced"] or state["Ready"])
-            and (new_id != 0)
-        )
+        if state["Failed"] or state["Invalid"]:
+            return False
+        # Orphan is valid (waiting for parent), or Synced/Ready with valid ID
+        return state["Orphan"] or ((state["Synced"] or state["Ready"]) and (new_id != 0))
 
     def _create_instance(self, response, klass=None, collection: bool = False):
         klass = klass or self._instance_klass
@@ -587,17 +591,14 @@ def entity_id(self, value):
 
     def get_id_from_crd(self):
         """Returns object id extracted from CRD."""
-        counter = 5
-        while counter > 0:
+        # 12 tries with fibonacci backoff: 1+1+2+3+5+8+13+21+34+55+89+144 ≈ 376 seconds (~6 min)
+        @backoff.on_predicate(backoff.fibo, lambda x: x is None, max_tries=12, jitter=None)
+        def _get_id():
             self.crd = self.crd.refresh()
             status = self.crd.as_dict()["status"]
-            ret_id = status.get(self.client.ID_NAME, None)
-            if ret_id:
-                return ret_id
-            time.sleep(20)
-            counter -= 1
+            return status.get(self.client.ID_NAME, None)
 
-        return None
+        return _get_id()
 
     def get_path(self):
         """

threescale_api_crd/resources.py

@@ -1,5 +1,6 @@
 """ Module with resources for CRD for Threescale client """
 
+import backoff
 import logging
 import copy
 import json
@@ -25,6 +26,54 @@
 from threescale_api_crd import constants
 
 LOG = logging.getLogger(__name__)
+BACKOFF_MAX_TRIES = 12
+
+
+class CRDNotReadyError(Exception):
+    """Raised when CRD is not yet ready or synced."""
+    pass
+
+
+def _is_crd_ready(crd):
+    """Check if CRD has status.conditions indicating it's ready to use.
+
+    Ready states:
+    - Synced=True or Ready=True
+    - Orphan=True (waiting for parent resource) with no Failed/Invalid
+    """
+    crd_dict = crd.as_dict()
+    status = crd_dict.get("status")
+    if not status:
+        return False
+    conditions = status.get("conditions", [])
+    state = {"Failed": False, "Invalid": False, "Synced": False, "Ready": False, "Orphan": False}
+    for cond in conditions:
+        cond_type = cond.get("type")
+        if cond_type in state:
+            state[cond_type] = cond.get("status") == "True"
+    # Fail if Failed or Invalid
+    if state["Failed"] or state["Invalid"]:
+        return False
+    # Ready if Synced, Ready, or Orphan (waiting for parent)
+    return state["Synced"] or state["Ready"] or state["Orphan"]
+
+
+def _crd_status_info(crd, crd_type: str) -> str:
+    """Generate descriptive error message for CRD not ready state."""
+    crd_dict = crd.as_dict()
+    name = crd_dict.get("metadata", {}).get("name", "unknown")
+    namespace = crd_dict.get("metadata", {}).get("namespace", "unknown")
+    status = crd_dict.get("status")
+    if not status:
+        return f"{crd_type} '{name}' in namespace '{namespace}' has no status"
+    conditions = status.get("conditions", [])
+    if not conditions:
+        return f"{crd_type} '{name}' in namespace '{namespace}' has no conditions"
+    cond_summary = ", ".join(
+        f"{c.get('type')}={c.get('status')}" + (f" ({c.get('message')})" if c.get("message") else "")
+        for c in conditions
+    )
+    return f"{crd_type} '{name}' in namespace '{namespace}' not ready: {cond_summary}"
 
 
 class Services(DefaultClientCRD, threescale_api.resources.Services):
@@ -1979,6 +2028,13 @@ def __init__(self, entity_name="system_name", **kwargs):
                 for cey, walue in constants.KEYS_SERVICE.items():
                     if key == walue:
                         entity[cey] = value
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "Service"))
+
+            _wait()
             entity["id"] = crd.as_dict().get("status").get(Services.ID_NAME)
             # add ids to cache
             if entity["id"] and entity[entity_name]:
@@ -2040,6 +2096,13 @@ def __init__(self, **kwargs):
             crd = kwargs.pop("crd")
             self.spec_path = []
             entity = {}
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "Proxy"))
+
+            _wait()
             # there is no attribute which can simulate Proxy id, service id should be used
             entity["id"] = crd.as_dict().get("status").get(Services.ID_NAME)
             # apicastHosted or ApicastSelfManaged
@@ -2335,6 +2398,13 @@ def __init__(self, entity_name="system_name", **kwargs):
                 for cey, walue in constants.KEYS_ACTIVE_DOC.items():
                     if key == walue:
                         entity[cey] = value
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "ActiveDoc"))
+
+            _wait()
             entity["id"] = crd.as_dict().get("status").get(ActiveDocs.ID_NAME)
             if "service_id" in entity:
                 ide = Service.system_name_to_id.get(entity["service_id"], None)
@@ -2374,6 +2444,13 @@ def __init__(self, entity_name="name", **kwargs):
                 for cey, walue in constants.KEYS_POLICY_REG.items():
                     if key == walue:
                         entity[cey] = value
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "PolicyRegistry"))
+
+            _wait()
             entity["id"] = crd.as_dict().get("status").get(PoliciesRegistry.ID_NAME)
             super().__init__(crd=crd, entity=entity, entity_name=entity_name, **kwargs)
         else:
@@ -2399,6 +2476,13 @@ def __init__(self, entity_name="system_name", **kwargs):
                 for cey, walue in constants.KEYS_BACKEND.items():
                     if key == walue:
                         entity[cey] = value
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "Backend"))
+
+            _wait()
             entity["id"] = crd.as_dict().get("status").get(Backends.ID_NAME)
 
             super().__init__(crd=crd, entity=entity, entity_name=entity_name, **kwargs)
@@ -2526,6 +2610,13 @@ def __init__(self, entity_name="system_name", **kwargs):
                 for cey, walue in constants.KEYS_BACKEND_USAGE.items():
                     if key == walue:
                         entity[cey] = value
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "BackendUsage"))
+
+            _wait()
             entity["service_id"] = int(
                 crd.as_dict().get("status", {}).get(Services.ID_NAME, 0)
             )
@@ -2644,6 +2735,13 @@ def __init__(self, entity_name="org_name", **kwargs):
                 for cey, walue in constants.KEYS_ACCOUNT.items():
                     if key == walue:
                         entity[cey] = value
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "Account"))
+
+            _wait()
             status = crd.as_dict().get("status", None)
             if status:
                 entity["id"] = status.get(Accounts.ID_NAME)
@@ -2705,6 +2803,13 @@ def __init__(self, entity_name="username", **kwargs):
                 for cey, walue in constants.KEYS_ACCOUNT_USER.items():
                     if key == walue:
                         entity[cey] = value
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "AccountUser"))
+
+            _wait()
             status = crd.as_dict().get("status", None)
             if status:
                 entity["id"] = status.get(AccountUsers.ID_NAME)
@@ -2768,6 +2873,13 @@ def __init__(self, entity_name="name", **kwargs):
                 for cey, walue in constants.KEYS_POLICY.items():
                     if key == walue:
                         entity[cey] = value
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "Policy"))
+
+            _wait()
             entity["service_id"] = int(
                 crd.as_dict().get("status", {}).get(Services.ID_NAME, 0)
             )
@@ -2814,6 +2926,13 @@ def __init__(self, entity_name="name", **kwargs):
                 for cey, walue in constants.KEYS_OPEN_API.items():
                     if key == walue:
                         entity[cey] = value
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "OpenApi"))
+
+            _wait()
             status = crd.as_dict().get("status")
             entity["id"] = status.get(OpenApis.ID_NAME)
             entity["productResourceName"] = status.get("productResourceName", {}).get(
@@ -3115,6 +3234,13 @@ def __init__(self, entity_name="name", **kwargs):
                 for cey, walue in constants.KEYS_APPLICATION.items():
                     if key == walue:
                         entity[cey] = value
+
+            @backoff.on_exception(backoff.fibo, CRDNotReadyError, max_tries=BACKOFF_MAX_TRIES, jitter=None)
+            def _wait():
+                if not _is_crd_ready(crd):
+                    raise CRDNotReadyError(_crd_status_info(crd, "Application"))
+
+            _wait()
             status = crd.as_dict().get("status")
             entity["id"] = status.get(Applications.ID_NAME)
             entity["state"] = status.get("state")