Change method for fetching visible projects and issue types in __heartbeat__

jbi/configuration.py

@@ -2,7 +2,6 @@
 Parsing and validating the YAML configuration occurs within this module
 """
 import logging
-from functools import lru_cache
 
 from pydantic import ValidationError
 from pydantic_yaml import parse_yaml_raw_as
@@ -18,12 +17,6 @@ class ConfigError(Exception):
     """Error when an exception occurs during processing config"""
 
 
-@lru_cache
-def get_actions() -> Actions:
-    """Load actions from file determined by ENV name"""
-    return get_actions_from_file(f"config/config.{settings.env}.yaml")
-
-
 def get_actions_from_file(jbi_config_file: str) -> Actions:
     """Convert and validate YAML configuration to `Action` objects"""
     try:
@@ -34,3 +27,11 @@ def get_actions_from_file(jbi_config_file: str) -> Actions:
     except ValidationError as exception:
         logger.exception(exception)
         raise ConfigError("Errors exist.") from exception
+
+
+def get_actions(env=settings.env):
+    """Load actions from file determined by ENV name"""
+    return get_actions_from_file(f"config/config.{env}.yaml")
+
+
+ACTIONS = get_actions()

jbi/models.py

@@ -138,13 +138,21 @@ def validate_actions(cls, actions: list[Action]):
         """
         Inspect the list of actions:
          - Validate that lookup tags are uniques
+         - Ensure we haven't exceeded our maximum configured project limit (see error below)
          - If the action's bugzilla_user_id is "tbd", emit a warning.
         """
         tags = [action.whiteboard_tag.lower() for action in actions]
         duplicated_tags = [t for i, t in enumerate(tags) if t in tags[:i]]
         if duplicated_tags:
             raise ValueError(f"actions have duplicated lookup tags: {duplicated_tags}")
 
+        if len(tags) > 50:
+            raise ValueError(
+                "The Jira client's `paginated_projects` method assumes we have "
+                "up to 50 projects configured. Adjust that implementation before "
+                "removing this validation check."
+            )
+
         for action in actions:
             if action.bugzilla_user_id == "tbd":
                 warnings.warn(

jbi/router.py

@@ -9,14 +9,14 @@
 from fastapi.responses import HTMLResponse
 from fastapi.templating import Jinja2Templates
 
-from jbi.configuration import get_actions
+from jbi.configuration import ACTIONS
 from jbi.environment import Settings, get_settings, get_version
 from jbi.models import Actions, BugzillaWebhookRequest
 from jbi.runner import IgnoreInvalidRequestError, execute_action
 from jbi.services import bugzilla, jira
 
 SettingsDep = Annotated[Settings, Depends(get_settings)]
-ActionsDep = Annotated[Actions, Depends(get_actions)]
+ActionsDep = Annotated[Actions, Depends(lambda: ACTIONS)]
 VersionDep = Annotated[dict, Depends(get_version)]
 BugzillaServiceDep = Annotated[bugzilla.BugzillaService, Depends(bugzilla.get_service)]
 JiraServiceDep = Annotated[jira.JiraService, Depends(jira.get_service)]
@@ -106,8 +106,7 @@ def get_bugzilla_webhooks(bugzilla_service: BugzillaServiceDep):
 @router.get("/jira_projects/")
 def get_jira_projects(jira_service: JiraServiceDep):
     """API for viewing projects that are currently accessible by API"""
-    visible_projects: list[dict] = jira_service.fetch_visible_projects()
-    return [project["key"] for project in visible_projects]
+    return jira_service.fetch_visible_projects()
 
 
 SRC_DIR = Path(__file__).parent

jbi/services/jira.py

@@ -11,7 +11,7 @@
 import json
 import logging
 from functools import lru_cache
-from typing import TYPE_CHECKING, Any, Iterable, Optional
+from typing import TYPE_CHECKING, Any, Collection, Iterable, Optional
 
 import requests
 from atlassian import Jira
@@ -94,7 +94,6 @@ def raise_for_status(self, *args, **kwargs):
 
     get_server_info = instrumented_method(Jira.get_server_info)
     get_project_components = instrumented_method(Jira.get_project_components)
-    projects = instrumented_method(Jira.projects)
     update_issue = instrumented_method(Jira.update_issue)
     update_issue_field = instrumented_method(Jira.update_issue_field)
     set_issue_status = instrumented_method(Jira.set_issue_status)
@@ -103,17 +102,55 @@ def raise_for_status(self, *args, **kwargs):
     get_project = instrumented_method(Jira.get_project)
 
     @instrumented_method
-    def permitted_projects(self, permissions: Iterable):
+    def paginated_projects(
+        self,
+        included_archived=None,
+        expand=None,
+        url=None,
+        keys: Optional[Collection[str]] = None,
+    ):
+        """Returns a paginated list of projects visible to the user.
+
+        https://developer.atlassian.com/cloud/jira/platform/rest/v2/api-group-projects/#api-rest-api-2-project-search-get
+
+        We've patched this method of the Jira client to accept the `keys` param.
+        """
+
+        if not self.cloud:
+            raise ValueError(
+                "``projects_from_cloud`` method is only available for Jira Cloud platform"
+            )
+
+        params = {}
+
+        if keys is not None:
+            if len(keys) > 50:
+                raise ValueError("Up to 50 project keys can be provided.")
+            params["keys"] = list(keys)
+
+        if included_archived:
+            params["includeArchived"] = included_archived
+        if expand:
+            params["expand"] = expand
+        page_url = url or self.resource_url("project/search")
+        is_url_absolute = bool(page_url.lower().startswith("http"))
+        return self.get(page_url, params=params, absolute=is_url_absolute)
+
+    @instrumented_method
+    def permitted_projects(self, permissions: Optional[Iterable] = None) -> list[dict]:
         """Fetches projects that the user has the required permissions for
 
         https://developer.atlassian.com/cloud/jira/platform/rest/v2/api-group-permissions/#api-rest-api-2-permissions-project-post
         """
+        if permissions is None:
+            permissions = []
 
         response = self.post(
             "/rest/api/2/permissions/project",
             json={"permissions": list(JIRA_REQUIRED_PERMISSIONS)},
         )
-        return response["projects"]
+        projects: list[dict] = response["projects"]
+        return projects
 
 
 class JiraService:
@@ -122,11 +159,11 @@ class JiraService:
     def __init__(self, client) -> None:
         self.client = client
 
-    def fetch_visible_projects(self) -> list[dict]:
+    def fetch_visible_projects(self) -> list[str]:
         """Return list of projects that are visible with the configured Jira credentials"""
 
-        projects: list[dict] = self.client.projects(included_archived=None)
-        return projects
+        projects = self.client.permitted_projects()
+        return [project["key"] for project in projects]
 
     def check_health(self, actions: Actions) -> ServiceHealth:
         """Check health for Jira Service"""
@@ -149,14 +186,12 @@ def check_health(self, actions: Actions) -> ServiceHealth:
 
     def _all_projects_visible(self, actions: Actions) -> bool:
         try:
-            visible_projects = {
-                project["key"] for project in self.fetch_visible_projects()
-            }
+            visible_projects = self.fetch_visible_projects()
         except requests.HTTPError:
             logger.exception("Error fetching visible Jira projects")
             return False
 
-        missing_projects = actions.configured_jira_projects_keys - visible_projects
+        missing_projects = actions.configured_jira_projects_keys - set(visible_projects)
         if missing_projects:
             logger.error(
                 "Jira projects %s are not visible with configured credentials",
@@ -232,7 +267,10 @@ def _check_project_components(self, action):
         return True
 
     def _all_project_issue_types_exist(self, actions: Actions):
-        projects = self.client.projects(included_archived=None, expand="issueTypes")
+        paginated_project_response = self.client.paginated_projects(
+            expand="issueTypes", keys=actions.configured_jira_projects_keys
+        )
+        projects = paginated_project_response["values"]
         issue_types_by_project = {
             project["key"]: {issue_type["name"] for issue_type in project["issueTypes"]}
             for project in projects

pyproject.toml

@@ -39,6 +39,7 @@ responses = "^0.24.1"
 httpx = "^0.25.2"
 factory-boy = "^3.3.0"
 pytest-factoryboy = "^2.6.0"
+pytest-mock = "^3.12.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

tests/conftest.py

@@ -83,7 +83,6 @@ def settings():
 
 @pytest.fixture(autouse=True)
 def actions():
-    get_actions.cache_clear()
     return get_actions()
 
 

tests/unit/test_configuration.py

@@ -1,8 +1,6 @@
-from unittest import mock
-
 import pytest
 
-from jbi import configuration, environment
+from jbi import configuration
 
 
 def test_mock_jbi_files():
@@ -22,8 +20,10 @@ def test_actual_jbi_files():
     )
 
 
-def test_filename_uses_env():
-    configuration.get_actions.cache_clear()
-    with mock.patch("jbi.configuration.get_actions_from_file") as mocked:
-        configuration.get_actions()
-    mocked.assert_called_with("config/config.local.yaml")
+def test_filename_uses_env(mocker, actions, settings):
+    get_actions_from_file_spy = mocker.spy(configuration, "get_actions_from_file")
+    assert settings.env == "local"
+
+    configuration.get_actions()
+
+    get_actions_from_file_spy.assert_called_with("config/config.local.yaml")

tests/unit/test_models.py

@@ -172,3 +172,9 @@ def test_payload_changes_list(webhook_event_change_factory, webhook_event_factor
         "status",
         "assignee",
     ]
+
+
+def test_max_configured_projects_raises_error(action_factory):
+    actions = [action_factory(whiteboard_tag=str(i)) for i in range(51)]
+    with pytest.raises(pydantic.ValidationError):
+        Actions(root=actions)