Testin resync slack_messages

slackbot/__init__.py

@@ -1,6 +1,6 @@
 from functools import lru_cache
 
-__version__ = "0.1.0"
+__version__ = "0.1.1"
 
 
 @lru_cache

slackbot/admin.py

@@ -1,26 +1,114 @@
 from django.contrib import admin
 from django.utils.html import format_html
-
+from django.utils.safestring import mark_safe
+from logbasecommand.base import LogBaseCommand
 from . import get_user_model
+from slackbot import models
+import json
 
 
 @admin.register(get_user_model())
 class UserAdmin(admin.ModelAdmin):
-    list_display = ['ext_id', 'username', 'name', 'email', 'active', 'is_bot', 'is_admin', 'get_photo', 'last_seen']
-    list_filter = ('active', 'is_bot', 'is_admin')
-    search_fields = ['username', 'name', 'email', 'ext_id']
-    readonly_fields = ['ext_id', 'username', 'name', 'email', 'active', 'is_bot', 'is_admin', 'get_photo', 'last_seen']
-    exclude = ['photo', 'photo_thumb']
+    list_display = [
+        "ext_id",
+        "username",
+        "name",
+        "email",
+        "active",
+        "is_bot",
+        "is_admin",
+        "get_photo",
+        "last_seen",
+    ]
+    list_filter = ("active", "is_bot", "is_admin")
+    search_fields = ["username", "name", "email", "ext_id"]
+    readonly_fields = [
+        "ext_id",
+        "username",
+        "name",
+        "email",
+        "active",
+        "is_bot",
+        "is_admin",
+        "get_photo",
+        "last_seen",
+    ]
+    exclude = ["photo", "photo_thumb"]
 
     def get_photo(self, obj):
         if obj.photo:
             return format_html(
-                '<a href="{}" target="_blank"><img src="{}" style="width:50px;"></a>', obj.photo, obj.photo_thumb
+                '<a href="{}" target="_blank"><img src="{}" style="width:50px;"></a>',
+                obj.photo,
+                obj.photo_thumb,
             )
         return None
 
-    get_photo.short_description = 'Photo'
+    get_photo.short_description = "Photo"
     get_photo.allow_tags = True
 
     def has_add_permission(self, _):
         return False
+
+
+@admin.register(models.SlackMessage)
+class SlackMessageAdmin(admin.ModelAdmin, LogBaseCommand):
+    list_display = (
+        "channel",
+        "message_from",
+        "reply_count",
+        "reply_users_list",
+        "reply_users_count",
+        "reactions_pretty",
+        "type",
+        "user_team",
+    )
+
+    readonly_fields = (
+        "channel",
+        "channel_id",
+        "client_msg_id",
+        "reactions_pretty",
+        "reply_count",
+        "reply_users_list",
+        "reply_users_count",
+        "team",
+        "text",
+        "thread_timestamp",
+        "time_stamp",
+        "type",
+        "user",
+        "message_from",
+        "user_team",
+        "thread_message_pretty",
+    )
+    exclude = ("reactions", "reply_users", "thread_message")
+    list_filter = ("channel", "message_from")
+
+    def reply_users_list(self, obj):
+        return ", ".join(obj.reply_users)
+
+    def thread_message_pretty(self, obj):
+        if not obj.thread_message:
+            return "-"
+        formatted_json = json.dumps(obj.thread_message, indent=4, ensure_ascii=False)
+        return mark_safe(f"<pre>{formatted_json}</pre>")
+
+    def reactions_pretty(self, obj):
+        if not obj.reactions:
+            return "-"
+        data = json.loads(obj.reactions) if isinstance(obj.reactions, str) else obj.reactions
+        formatted_output = []
+        for item in data:
+            name = item.get("name", "N/A")
+            count = item.get("count", 0)
+            users = item.get("users", [])
+            formatted_output.append(f"<b>{name}</b> (Count: {count})<br>Users: {users}")
+
+        return format_html("<br><br>".join(formatted_output))
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False

slackbot/apps.py

@@ -7,20 +7,21 @@
 APP_SETTINGS = dict(
     BOT_TOKEN=None,
     APP_TOKEN=None,
-    USER_MODEL='slackbot.User',
+    SLACK_CHANNELS={"test_channel": "1"},
+    USER_MODEL="slackbot.User",
 )
 
 
 class SlackbotConfig(AppConfig):
-    name = 'slackbot'
-    default_auto_field = 'django.db.models.AutoField'
+    name = "slackbot"
+    default_auto_field = "django.db.models.AutoField"
 
     def __init__(self, app_name: str, app_module) -> None:
         super().__init__(app_name, app_module)
         for k, v in APP_SETTINGS.items():
-            _k = 'SLACKBOT_%s' % k
+            _k = "SLACKBOT_%s" % k
             if not hasattr(settings, _k):
                 setattr(settings, _k, v)
 
     def ready(self):
-        autodiscover_modules('slack')
+        autodiscover_modules("slack")

slackbot/management/commands/resync_slack_messages.py

@@ -0,0 +1,117 @@
+import time
+from datetime import datetime
+
+from django.conf import settings
+from django.utils import timezone
+from slack_sdk import WebClient
+from slack_sdk.errors import SlackApiError
+
+from logbasecommand.base import LogBaseCommand
+from slackbot.models import SlackMessage, User
+
+
+class Command(LogBaseCommand):
+    help = "Get all the details and messages from Slack from the channels in scope"
+
+    def handle(self, *args, **options) -> None:
+        client = WebClient(token=settings.SLACKBOT_BOT_TOKEN)
+        timestamp_limit = time.mktime(time.strptime("2024-01-01", "%Y-%m-%d"))
+        next_cursor = None
+
+        for channel, channel_id in settings.SLACKBOT_SLACK_CHANNELS.items():
+            next_cursor = None
+            while True:
+                try:
+                    response = client.conversations_history(
+                        channel=channel_id,
+                        cursor=next_cursor,
+                        limit=20,
+                        oldest=timestamp_limit,
+                    )
+                    last_h = 0
+                    for message in response["messages"]:
+                        thread_response = None
+                        aware_thread_timestamp = None
+                        time_stamp = datetime.fromtimestamp(float(message["ts"]))
+                        aware_timestamp = timezone.make_aware(time_stamp, timezone.get_default_timezone())
+                        now_h = time_stamp.strftime("%Y-%m-%d %H")
+                        if last_h != now_h:
+                            last_h = now_h
+
+                        if message.get("bot_id"):
+                            continue
+
+                        if aware_timestamp < timezone.now() - timezone.timedelta(days=365):
+                            continue
+
+                        replies = []
+                        if "thread_ts" in message:
+                            thread_response = client.conversations_replies(channel=channel_id, ts=message["thread_ts"])
+                            thread_response = thread_response["messages"][1:]
+                            thread_timestamp = datetime.fromtimestamp(float(message["thread_ts"]))
+                            aware_thread_timestamp = timezone.make_aware(
+                                thread_timestamp, timezone.get_default_timezone()
+                            )
+                        if thread_response is not None:
+                            for each in thread_response:
+                                user = User.objects.filter(ext_id=each.get("user", "")).first()
+                                if user:
+                                    user = user.name
+                                else:
+                                    user = "Unknown"
+                                replies.append({"user": user, "text": each["text"]})
+                        reactions = []
+                        for each in message.get("reactions", ""):
+                            user = User.objects.filter(ext_id__in=each["users"]).first()
+                            if user:
+                                user = user.name
+                            else:
+                                user = "Unknown"
+                            reactions.append(
+                                {
+                                    "name": each["name"],
+                                    "users": user,
+                                    "count": each["count"],
+                                }
+                            )
+
+                        reply_users = list(
+                            User.objects.filter(ext_id__in=message.get("reply_users", "")).values_list(
+                                "name", flat=True
+                            )
+                        )
+                        SlackMessage.objects.update_or_create(
+                            ts=message["ts"],
+                            channel=channel,
+                            defaults={
+                                "channel_id": channel_id,
+                                "client_msg_id": message.get("client_msg_id", ""),
+                                "time_stamp": aware_timestamp,
+                                "reactions": reactions,
+                                "reply_count": message.get("reply_count", 0),
+                                "reply_users": reply_users,
+                                "reply_users_count": message.get("reply_users_count", 0),
+                                "team": message.get("team", ""),
+                                "text": message["text"],
+                                "thread_timestamp": aware_thread_timestamp,
+                                "type": message["type"],
+                                "user": message["user"],
+                                "message_from": message.get("user_profile", {}).get("display_name", "Unknown User"),
+                                "user_team": message.get("user_team", ""),
+                                "thread_message": replies,
+                            },
+                        )
+
+                    next_cursor = response.get("response_metadata", {}).get("next_cursor")
+                    if not next_cursor:
+                        break
+                except SlackApiError as e:
+                    if e.response["error"] == "ratelimited":
+                        retry_after = int(e.response.headers["Retry-After"])
+                        self.log_exception("Rate limit hit. Retrying.... %s")
+                        time.sleep(retry_after)
+                    else:
+                        self.log_exception("Error fetching history: %s", e.response["error"])
+                        break
+
+        SlackMessage.objects.filter(time_stamp__lt=timezone.now() - timezone.timedelta(days=365)).delete()

slackbot/migrations/0004_slack_messages.py

@@ -0,0 +1,51 @@
+# Generated by Django 4.2.3 on 2025-02-04 13:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [("slackbot", "0003_auto_20210915_1137")]
+
+    operations = [
+        migrations.CreateModel(
+            name="SlackMessage",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("channel", models.CharField(max_length=128)),
+                ("ts", models.CharField(max_length=32)),
+                ("channel_id", models.CharField(max_length=32)),
+                ("client_msg_id", models.CharField(max_length=64)),
+                ("reactions", models.JSONField(blank=True)),
+                ("reply_count", models.IntegerField(blank=True)),
+                ("reply_users", models.JSONField(blank=True, default=list)),
+                ("reply_users_count", models.IntegerField(blank=True)),
+                ("team", models.CharField(max_length=32)),
+                ("text", models.TextField()),
+                ("thread_timestamp", models.DateTimeField(blank=True, null=True)),
+                ("time_stamp", models.DateTimeField()),
+                ("type", models.CharField(max_length=32)),
+                ("user", models.CharField(max_length=32)),
+                ("message_from", models.CharField(max_length=128)),
+                ("user_team", models.CharField(max_length=32)),
+                ("thread_message", models.JSONField(blank=True, null=True)),
+            ],
+            options={
+                "verbose_name": "Slack Message",
+                "verbose_name_plural": "Slack Messages",
+            },
+        ),
+        migrations.AddConstraint(
+            model_name="slackmessage",
+            constraint=models.UniqueConstraint(
+                fields=("ts", "channel"), name="unique_ts_channel"
+            ),
+        ),
+    ]