feat: sentry api

add sentry api client and pull_sentry_project_slug task

Author: Kl0ven

controller/sentry/admin.py

@@ -31,6 +31,7 @@ class AppAdmin(
 
     list_display = [
         "reference",
+        "sentry_project_slug",
         "last_seen",
         "default_sample_rate",
         "active_sample_rate",
@@ -59,6 +60,13 @@ class AppAdmin(
                 )
             },
         ],
+        [
+            "Sentry",
+            {
+                "classes": ("collapse", "open"),
+                "fields": ("sentry_project_slug",),
+            },
+        ],
         [
             "WSGI",
             {

controller/sentry/migrations/0007_app_sentry_project_slug.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-02-01 11:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sentry", "0006_alter_app_options"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="app",
+            name="sentry_project_slug",
+            field=models.CharField(blank=True, max_length=50, null=True),
+        ),
+    ]

controller/sentry/models.py

@@ -28,6 +28,9 @@ class App(models.Model):
     active_sample_rate = models.FloatField(default=settings.DEFAULT_SAMPLE_RATE)
     active_window_end = models.DateTimeField(null=True, blank=True)
 
+    # Sentry Api
+    sentry_project_slug = models.CharField(max_length=50, null=True, blank=True)
+
     # WSGI
     wsgi_ignore_path = ArrayField(
         models.CharField(max_length=50, blank=True),
@@ -58,6 +61,12 @@ def get_metric(self, metric_type: MetricType):
         prefix = metric_type.value.lower()
         return getattr(self, f"{prefix}_collect_metrics"), getattr(self, f"{prefix}_metrics")
 
+    def get_sentry_id(self):
+        res = self.reference.split("_")
+        if len(res) != 3:
+            return None
+        return res[0]
+
     class Meta:
         permissions = [
             ("bump_sample_rate_app", "Can bump sample rate"),

controller/sentry/tasks.py

@@ -1,4 +1,5 @@
 from datetime import timedelta
+from itertools import chain
 
 from celery import shared_task
 from celery.utils.log import get_task_logger
@@ -7,6 +8,7 @@
 from django.utils import timezone
 
 from controller.sentry.models import App
+from controller.sentry.webservices.sentry import PaginatedSentryClient
 
 LOGGER = get_task_logger(__name__)
 
@@ -24,3 +26,21 @@ def prune_inactive_app() -> None:
 def close_window() -> None:
     apps = App.objects.filter(active_window_end__lt=timezone.now())
     apps.update(active_sample_rate=F("default_sample_rate"), active_window_end=None)
+
+
+@shared_task()
+def pull_sentry_project_slug() -> None:
+    client = PaginatedSentryClient()
+    apps = App.objects.filter(sentry_project_slug__isnull=True)
+
+    apps_by_id = {app.get_sentry_id(): app for app in apps}
+
+    modified_apps = []
+    for project in chain.from_iterable(client.list_projects()):
+        _id = project["id"]
+        if _id in apps_by_id:
+            app = apps_by_id[_id]
+            app.sentry_project_slug = project["slug"]
+            modified_apps.append(app)
+
+    App.objects.bulk_update(modified_apps, ["sentry_project_slug"])

controller/sentry/tests/test_models.py

@@ -14,3 +14,10 @@ def test_app_model_merge():
     collect, metrics = app.get_metric(MetricType.CELERY)
     assert not collect
     assert metrics == {"test1": 5, "test": 6}
+
+
+def test_app_model_get_sentry_id():
+    app = App(reference="abc")
+    assert app.get_sentry_id() is None
+    app = App(reference="123_prod_uwsgi")
+    assert app.get_sentry_id() == "123"

controller/sentry/tests/test_sentry_client.py

@@ -0,0 +1,96 @@
+from datetime import datetime, timedelta
+from unittest.mock import MagicMock, call, patch
+
+import pytest
+from requests.exceptions import HTTPError
+
+from controller.sentry.webservices.sentry import BearerAuth, PaginatedSentryClient
+
+
+class Response:
+    def __init__(self, status_code, data, links=None, headers=None) -> None:
+        self.status_code = status_code
+        self.data = data
+        self.links = links if links else {}
+        self.headers = headers if headers else {}
+
+    def json(self):
+        return self.data
+
+    def raise_for_status(self):
+        if self.status_code > 399:
+            raise HTTPError
+
+
+def test_auth():
+    auth = BearerAuth("my_token")
+    request_mock = MagicMock()
+    auth(request_mock)
+
+    request_mock.headers.__setitem__.assert_called_once_with("authorization", "Bearer my_token")
+
+
+@patch("controller.sentry.webservices.sentry.request")
+def test_client(mock_request: MagicMock):
+    client = PaginatedSentryClient()
+
+    res = client.list_projects()
+    mock_request.assert_not_called()
+    return_value = [
+        Response(200, [object(), object()], links={"next": {"results": True, "url": "http://next.sentry"}}),
+        Response(200, [object(), object()]),
+    ]
+
+    mock_request.side_effect = return_value
+
+    for chunk, expected in zip(res, return_value):
+        assert chunk == expected.data
+
+    call_1 = call("GET", "https://sentry.io/api/0/projects/", timeout=20, auth=client.auth)
+    call_2 = call("GET", "http://next.sentry", timeout=20, auth=client.auth)
+    mock_request.assert_has_calls((call_1, call_2))
+
+
+@patch("controller.sentry.webservices.sentry.request")
+def test_client_rate_limited(mock_request: MagicMock):
+    client = PaginatedSentryClient()
+
+    res = client.list_projects()
+    mock_request.assert_not_called()
+    reset = datetime.now() + timedelta(seconds=5)
+    return_value = [
+        Response(429, [object(), object()], headers={"x-sentry-rate-limit-reset": reset.timestamp()}),
+        Response(500, [object(), object()]),
+    ]
+
+    mock_request.side_effect = return_value
+
+    with pytest.raises(HTTPError):
+        next(res)
+
+    call_1 = call("GET", "https://sentry.io/api/0/projects/", timeout=20, auth=client.auth)
+    call_2 = call("GET", "https://sentry.io/api/0/projects/", timeout=20, auth=client.auth)
+    mock_request.assert_has_calls((call_1, call_2))
+
+
+@patch("controller.sentry.webservices.sentry.request")
+def test_client_rate_limited_rest_in_past(mock_request: MagicMock):
+    client = PaginatedSentryClient()
+
+    res = client.list_projects()
+    mock_request.assert_not_called()
+
+    reset = datetime.now() - timedelta(seconds=5)
+    return_value = [
+        Response(429, [object(), object()], headers={"x-sentry-rate-limit-reset": reset.timestamp()}),
+        Response(500, [object(), object()]),
+    ]
+
+    mock_request.side_effect = return_value
+
+    with pytest.raises(HTTPError):
+        next(res)
+
+    call_1 = call("GET", "https://sentry.io/api/0/projects/", timeout=20, auth=client.auth)
+    call_2 = call("GET", "https://sentry.io/api/0/projects/", timeout=20, auth=client.auth)
+    mock_request.assert_has_calls((call_1, call_2))

controller/sentry/tests/test_tasks.py

@@ -0,0 +1,61 @@
+from datetime import timedelta
+from unittest.mock import MagicMock, patch
+
+import pytest
+from django.conf import settings
+from django.utils import timezone
+
+from controller.sentry.models import App
+from controller.sentry.tasks import (
+    close_window,
+    prune_inactive_app,
+    pull_sentry_project_slug,
+)
+
+
+@pytest.mark.django_db
+def test_prune_inactive_app():
+    app = App(reference="abc")
+    app.save()
+    prune_inactive_app()
+    assert App.objects.filter(reference=app.reference).exists()
+
+    app.last_seen = timezone.now() - timedelta(days=settings.APP_AUTO_PRUNE_MAX_AGE_DAY + 1)
+    app.save()
+    prune_inactive_app()
+    assert not App.objects.filter(reference=app.reference).exists()
+
+
+@pytest.mark.django_db
+def test_close_window():
+    tomorrow = timezone.now() + timedelta(days=1)
+    app = App(reference="abc1", active_window_end=tomorrow, active_sample_rate=1)
+    app.save()
+    close_window()
+    app.refresh_from_db()
+    assert app.active_sample_rate == 1
+    assert app.active_window_end == tomorrow
+
+    app = App(reference="abc2", active_window_end=timezone.now(), active_sample_rate=1)
+    app.save()
+    close_window()
+    app.refresh_from_db()
+    assert app.active_sample_rate == app.default_sample_rate
+    assert app.active_window_end is None
+
+
+@patch("controller.sentry.tasks.PaginatedSentryClient")
+@pytest.mark.django_db
+def test_pull_sentry_project_slug(client_mock: MagicMock):
+    client_mock.return_value.list_projects.return_value = [
+        [],
+        [{"id": "123", "slug": "test"}],
+        [{"id": "1235", "slug": "test2"}],
+    ]
+    app = App(reference="123_prod_wsgi")
+    app.save()
+    pull_sentry_project_slug()
+    client_mock.assert_called_once_with()
+
+    app.refresh_from_db()
+    assert app.sentry_project_slug == "test"

controller/sentry/tests/test_views.py

@@ -1,10 +1,8 @@
-from datetime import timedelta
 from unittest.mock import Mock, patch
 
 import pytest
 from django.conf import settings
 from django.urls import reverse
-from django.utils import timezone
 
 from controller.sentry.choices import MetricType
 from controller.sentry.models import App
@@ -22,29 +20,6 @@ def test_app_view_retrieve(client):
     assert not response.data["celery_collect_metrics"]
 
 
-@pytest.mark.django_db
-def test_app_view_retrieve_windows_end(client):
-    reference = "test"
-    app = App(reference=reference)
-    app.active_window_end = timezone.now() + timedelta(hours=5)
-    app.active_sample_rate = 1
-    app.default_sample_rate = 0.5
-    app.save()
-    url = reverse("sentry:apps-detail", kwargs={"pk": reference})
-    response = client.get(url)
-    assert response.status_code == 200
-    assert response.data["active_window_end"] is not None
-    assert response.data["active_sample_rate"] == 1
-
-    app.active_window_end = timezone.now() - timedelta(hours=5)
-    app.save()
-    url = reverse("sentry:apps-detail", kwargs={"pk": reference})
-    response = client.get(url)
-    assert response.status_code == 200
-    assert response.data["active_window_end"] is None
-    assert response.data["active_sample_rate"] == 0.5
-
-
 @patch("controller.sentry.views.cache")
 @pytest.mark.django_db
 def test_app_view_retrieve_panic(cache: Mock, client):

controller/sentry/utils.py

@@ -41,3 +41,12 @@ def invalidate_cache(path=""):
 # pylint: disable=unused-argument
 def is_panic_activated(request):
     return {"PANIC": cache.get(settings.PANIC_KEY)}
+
+
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]