Merge pull request #4942 from ministryofjustice/poc-repository-alerting

POC repository alerting

.github/workflows/job-alarm-for-old-poc-repositories.yml

@@ -0,0 +1,38 @@
+name: Alarm for Old POC Repositories
+
+on:
+  schedule:
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+
+env:
+  ADMIN_SLACK_TOKEN: ${{ secrets.ADMIN_SEND_TO_SLACK }}
+  GH_TOKEN: ${{ secrets.OPS_ENG_GENERAL_ADMIN_BOT_PAT }}
+
+jobs:
+  old-poc-repositories-alarm:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pipenv"
+
+      - name: Install Pipenv
+        run: |
+          pip install pipenv
+          pipenv install
+
+      - run: pipenv run python3 -m bin.alarm_for_old_poc_repositories
+
+      - name: Report failure to Slack
+        if: always()
+        uses: ravsamhq/notify-slack-action@472601e839b758e36c455b5d3e5e1a217d4807bd # 2.5.0
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "Failed: Alarm for Old POC Repositories"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

bin/alarm_for_old_poc_repositories.py

@@ -0,0 +1,40 @@
+import sys
+import os
+
+from services.github_service import GithubService
+from services.slack_service import SlackService
+from config.constants import ENTERPRISE, MINISTRY_OF_JUSTICE, SLACK_CHANNEL
+
+def construct_message(repositories):
+    intro = "The following POC GitHub Repositories persist:\n\n"
+
+    core_content = "\n".join([f"https://github.com/ministryofjustice/{repo} - {age} days old" for repo, age in repositories.items()])
+
+    action = "\n\nConsider if they are still required. If not, please archive them by removing them from the Terraform configuration: https://github.com/ministryofjustice/operations-engineering/tree/main/terraform/github/repositories/ministryofjustice"
+
+    return intro + core_content + action
+
+
+def alert_for_old_poc_repositories():
+    github_token = os.environ.get("GH_TOKEN")
+    slack_token = os.environ.get("ADMIN_SLACK_TOKEN")
+
+    if github_token is None:
+        print("No GH_TOKEN environment variable set")
+        sys.exit(1)
+
+    if slack_token is None:
+        print("No ADMIN_SLACK_TOKEN environment variable set")
+        sys.exit(1)
+
+    github_service = GithubService(github_token, MINISTRY_OF_JUSTICE, ENTERPRISE)
+    slack_service = SlackService(slack_token)
+
+    old_poc_repositories = github_service.get_old_poc_repositories()
+
+    if old_poc_repositories:
+        slack_service.send_message_to_plaintext_channel_name(construct_message(old_poc_repositories), SLACK_CHANNEL)
+
+
+if __name__ == "__main__":
+    alert_for_old_poc_repositories()

services/github_service.py

@@ -2,7 +2,7 @@
 
 import json
 from calendar import timegm
-from datetime import date, datetime, timedelta
+from datetime import date, datetime, timedelta, timezone
 from time import gmtime, sleep
 from typing import Any, Callable
 
@@ -1241,3 +1241,25 @@ def get_all_enterprise_members(self) -> list:
             all_users = all_users + [user.login for user in self.github_client_core_api.get_organization(org).get_members() if user.login not in all_users]
 
         return all_users
+
+    @retries_github_rate_limit_exception_at_next_reset_once
+    def calculate_repo_age(self, repo: str) -> list:
+        creation_date = self.github_client_core_api.get_repo(f"{self.organisation_name}/{repo}").created_at
+
+        age_in_days = (datetime.now(timezone.utc) - creation_date).days
+
+        return age_in_days
+
+    @retries_github_rate_limit_exception_at_next_reset_once
+    def get_old_poc_repositories(self) -> list:
+        poc_repositories = [repo['repo']['name'] for repo in self.get_paginated_list_of_repositories_per_topic("poc", None)['search']['repos']]
+
+        old_poc_repositories = {}
+        age_threshold = 30
+
+        for repo in poc_repositories:
+            age = self.calculate_repo_age(repo)
+            if age >= age_threshold:
+                old_poc_repositories[repo] = age
+
+        return old_poc_repositories

test/test_bin/test_alarm_for_old_poc_repositories.py

@@ -0,0 +1,69 @@
+import unittest
+from unittest.mock import patch, MagicMock
+
+from bin.alarm_for_old_poc_repositories import (
+    construct_message,
+    alert_for_old_poc_repositories
+)
+
+from services.github_service import GithubService
+from services.slack_service import SlackService
+
+
+class TestOldPOCGitHubRepositoriesAlerting(unittest.TestCase):
+
+    def test_construct_message(self):
+        test_payload = {"repo1": 51, "repo2": 60}
+
+        self.assertEqual(construct_message(test_payload), "The following POC GitHub Repositories persist:\n\nhttps://github.com/ministryofjustice/repo1 - 51 days old\nhttps://github.com/ministryofjustice/repo2 - 60 days old\n\nConsider if they are still required. If not, please archive them by removing them from the Terraform configuration: https://github.com/ministryofjustice/operations-engineering/tree/main/terraform/github/repositories/ministryofjustice")
+
+    @patch("gql.transport.aiohttp.AIOHTTPTransport.__new__", new=MagicMock)
+    @patch("gql.Client.__new__", new=MagicMock)
+    @patch("github.Github.__new__")
+    @patch("bin.alarm_for_old_poc_repositories.construct_message")
+    @patch.object(GithubService, "get_old_poc_repositories")
+    @patch.object(SlackService, "send_message_to_plaintext_channel_name")
+    @patch('os.environ')
+    def test_alert_for_old_poc_repositories_if_found(
+        self,
+        mock_env,
+        mock_send_message_to_plaintext_channel_name,
+        mock_get_old_poc_repositories,
+        mock_construct_message,
+        _mock_github_client_core_api
+    ):
+
+        mock_env.get.side_effect = lambda k: 'mock_token' if k in ['GH_TOKEN', 'ADMIN_SLACK_TOKEN'] else None
+        mock_get_old_poc_repositories.return_value = {"repo1": 51, "repo2": 60}
+        mock_construct_message.return_value = "The following POC GitHub Repositories persist:\n\nhttps://github.com/ministryofjustice/repo1 - 51 days old\nhttps://github.com/ministryofjustice/repo2 - 60 days old\n\nConsider if they are still required. If not, please archive them by removing them from the Terraform configuration: https://github.com/ministryofjustice/operations-engineering/tree/main/terraform/github/repositories/ministryofjustice"
+
+        alert_for_old_poc_repositories()
+
+        mock_get_old_poc_repositories.assert_called_once()
+        mock_send_message_to_plaintext_channel_name.assert_called_once_with("The following POC GitHub Repositories persist:\n\nhttps://github.com/ministryofjustice/repo1 - 51 days old\nhttps://github.com/ministryofjustice/repo2 - 60 days old\n\nConsider if they are still required. If not, please archive them by removing them from the Terraform configuration: https://github.com/ministryofjustice/operations-engineering/tree/main/terraform/github/repositories/ministryofjustice", "operations-engineering-alerts")
+
+    @patch("gql.transport.aiohttp.AIOHTTPTransport.__new__", new=MagicMock)
+    @patch("gql.Client.__new__", new=MagicMock)
+    @patch("github.Github.__new__")
+    @patch.object(GithubService, "get_old_poc_repositories")
+    @patch.object(SlackService, "send_message_to_plaintext_channel_name")
+    @patch('os.environ')
+    def test_alert_for_old_poc_repositories_if_not_found(
+        self,
+        mock_env,
+        mock_send_message_to_plaintext_channel_name,
+        mock_get_old_poc_repositories,
+        _mock_github_client_core_api
+    ):
+
+        mock_env.get.side_effect = lambda k: 'mock_token' if k in ['GH_TOKEN', 'ADMIN_SLACK_TOKEN'] else None
+        mock_get_old_poc_repositories.return_value = {}
+
+        alert_for_old_poc_repositories()
+
+        mock_get_old_poc_repositories.assert_called_once()
+        assert not mock_send_message_to_plaintext_channel_name.called
+
+
+if __name__ == '__main__':
+    unittest.main()

test/test_services/test_github_service.py

@@ -120,6 +120,7 @@ def test_sets_up_class(self,  mock_github_client_rest_api, mock_github_client_co
 class TestGithubServiceArchiveInactiveRepositories(unittest.TestCase):
 
     # Default for archiving a repository
+    # pylint: disable=R0917
     def __get_repository(self, last_active_date: datetime, created_at_date: datetime, archived: bool = False, fork: bool = False, repo_name: str = None, has_commits: bool = True) -> Mock:
         repository_to_consider_for_archiving = Mock(
             Repository, archived=archived, fork=fork)
@@ -1878,5 +1879,37 @@ def test_get_all_enterprise_members(self, mock_github_client_core_api):
         self.assertEqual(2, len(response))
 
 
+@patch("gql.transport.aiohttp.AIOHTTPTransport.__new__", new=MagicMock)
+@patch("gql.Client.__new__", new=MagicMock)
+@patch("github.Github.__new__")
+class TestGetOldPOCRepositories(unittest.TestCase):
+
+    @freeze_time("2024-10-24")
+    def test_calculate_repo_age(self, mock_github_client_core_api):
+        mock_github_client_core_api.return_value.get_repo().created_at = datetime.fromisoformat("2024-10-04 00:00:00+00:00")
+
+        response = GithubService("", ORGANISATION_NAME).calculate_repo_age("repo1")
+
+        self.assertEqual(20, response)
+
+    @patch.object(GithubService, "calculate_repo_age")
+    @patch.object(GithubService, "get_paginated_list_of_repositories_per_topic")
+    def test_get_old_poc_repositories_if_exist(self, mock_get_paginated_list_of_repositories_per_topic, mock_calculate_repo_age, _mock_github_client_core_api):
+        mock_get_paginated_list_of_repositories_per_topic.return_value = {'search': {'repos': [{'repo': {'name': 'operations-engineering-metadata-poc', 'isDisabled': False, 'isLocked': False, 'hasIssuesEnabled': True, 'repositoryTopics': {'edges': [{'node': {'topic': {'name': 'operations-engineering'}}}, {'node': {'topic': {'name': 'poc'}}}]}, 'collaborators': {'totalCount': 0}}}, {'repo': {'name': 'operations-engineering-unit-test-generator-poc', 'isDisabled': False, 'isLocked': False, 'hasIssuesEnabled': True, 'repositoryTopics': {'edges': [{'node': {'topic': {'name': 'operations-engineering'}}}, {'node': {'topic': {'name': 'poc'}}}]}, 'collaborators': {'totalCount': 0}}},], 'pageInfo': {'hasNextPage': False, 'endCursor': 'Y3Vyc29yOjQ='}}}
+        mock_calculate_repo_age.return_value = 30
+
+        response = GithubService("", ORGANISATION_NAME).get_old_poc_repositories()
+
+        self.assertEqual({"operations-engineering-metadata-poc": 30, "operations-engineering-unit-test-generator-poc": 30}, response)
+
+    @patch.object(GithubService, "get_paginated_list_of_repositories_per_topic")
+    def test_get_old_poc_repositories_if_not_exist(self, mock_get_paginated_list_of_repositories_per_topic, _mock_github_client_core_api):
+        mock_get_paginated_list_of_repositories_per_topic.return_value = {'search': {'repos': [], 'pageInfo': {'hasNextPage': False, 'endCursor': 'Y3Vyc29yOjQ='}}}
+
+        response = GithubService("", ORGANISATION_NAME).get_old_poc_repositories()
+
+        self.assertEqual({}, response)
+
+
 if __name__ == "__main__":
     unittest.main()