Merge pull request #317 from sysblok/feature/add-fill-posts-list-foca…

…lboard

Feature/add fill posts list focalboard

src/bot.py

@@ -118,6 +118,15 @@ def init_handlers(self):
             direct_message_only(self.manager_reply_handler("fill_posts_list_job")),
             "заполнить реестр постов",
         )
+
+        self.add_manager_handler(
+            "fill_posts_list_focalboard",
+            CommandCategories.REGISTRY,
+            direct_message_only(
+                self.manager_reply_handler("fill_posts_list_focalboard_job")
+            ),
+            "заполнить реестр постов из Focalboard",
+        )
         self.add_admin_handler(
             "send_editorial_report",
             CommandCategories.BROADCAST,

src/consts.py

@@ -116,6 +116,13 @@ class TrelloCustomFieldTypes(Enum):
     TEXT = "text"
     CHECKBOX = "checkbox"
     LIST = "list"
+    CREATED_AT = "createdTime"
+    CREATED_BY = "createdBy"
+    SELECT = "select"
+    MULTISELECT = "multiSelect"
+    MULTIPERSON = "multiPerson"
+    DATE = "date"
+    URL = "url"
 
 
 class TrelloCardFieldErrorAlias(Enum):

src/focalboard/focalboard_client.py

@@ -5,9 +5,10 @@
 
 import requests
 
+from ..consts import TrelloCustomFieldTypeAlias, TrelloCustomFieldTypes, TrelloListAlias
 from ..strings import load
-from ..utils.singleton import Singleton
 from ..trello import trello_objects as objects
+from ..utils.singleton import Singleton
 
 logger = logging.getLogger(__name__)
 
@@ -27,12 +28,11 @@ def get_boards_for_user(self, user_id=None):
         logger.debug(f"get_boards_for_user: {boards}")
         return boards
 
-    def get_lists(self, board_id):
-        # if not board_id:
-        #     board_id = self.board_id
+    def get_lists(self):
+        board_id = self.board_id
         # TODO make it more efficient
         # essentially all list information is already passed via boards handler
-        _, data = self._make_request(f"api/v2/teams/0/boards")
+        _, data = self._make_request("api/v2/teams/0/boards")
         list_data = [
             prop
             for prop in [board for board in data if board["id"] == board_id][0][
@@ -49,7 +49,7 @@ def get_lists(self, board_id):
         return lists
 
     def get_list(self, board_id, list_id):
-        _, data = self._make_request(f"api/v2/teams/0/boards")
+        _, data = self._make_request("api/v2/teams/0/boards")
         lists_data = [
             prop
             for prop in [board for board in data if board["id"] == board_id][0][
@@ -66,7 +66,7 @@ def get_list(self, board_id, list_id):
         return lst
 
     def _get_list_property(self, board_id):
-        _, data = self._make_request(f"api/v2/teams/0/boards")
+        _, data = self._make_request("api/v2/teams/0/boards")
         return [
             prop
             for prop in [board for board in data if board["id"] == board_id][0][
@@ -76,7 +76,7 @@ def _get_list_property(self, board_id):
         ][0]["id"]
 
     def _get_member_property(self, board_id):
-        _, data = self._make_request(f"api/v2/teams/0/boards")
+        _, data = self._make_request("api/v2/teams/0/boards")
         return [
             prop
             for prop in [board for board in data if board["id"] == board_id][0][
@@ -85,6 +85,64 @@ def _get_member_property(self, board_id):
             if prop["name"] == "Assignee"
         ][0]["id"]
 
+    def get_list_id_from_aliases(self, list_aliases):
+        list_ids = [
+            self.lists_config[alias]
+            for alias in list_aliases
+            if alias in self.lists_config
+        ]
+        if len(list_ids) != len(list_aliases):
+            logger.error(
+                f"list_ids not found for aliases: "
+                f"{[alias for alias in list_aliases if alias not in self.lists_config]}"
+            )
+        return list_ids
+
+    def _fill_alias_id_map(self, items, item_enum):
+        result = {}
+        for alias in item_enum:
+            suitable_items = [
+                item for item in items if item.name.startswith(load(alias.value))
+            ]
+            if len(suitable_items) > 1:
+                raise ValueError(
+                    f"Enum {item_enum.__name__} name {alias.value} is ambiguous!"
+                )
+            if len(suitable_items) > 0:
+                result[alias] = suitable_items[0].id
+        return result
+
+    def _fill_id_type_map(self, items, item_enum):
+        result = {}
+        for item in items:
+            result[item.id] = TrelloCustomFieldTypes(item.type)
+        return result
+
+    def get_board_custom_field_types(self):
+        board_id = self.board_id
+        _, data = self._make_request(f"api/v2/boards/{board_id}")
+        custom_field_types = [
+            objects.TrelloCustomFieldType.from_focalboard_dict(custom_field_type)
+            for custom_field_type in data["cardProperties"]
+        ]
+        logger.debug(f"get_board_custom_field_types: {custom_field_types}")
+        return custom_field_types
+
+    def get_board_custom_fields_dict(self, card_id):
+        custom_fields = self.get_board_custom_field_types()
+        custom_fields_dict = {}
+        for alias, type_id in self.custom_fields_config.items():
+            suitable_fields = [fld for fld in custom_fields if fld.id == type_id]
+            if len(suitable_fields) > 0:
+                custom_fields_dict[alias] = suitable_fields[0]
+        return custom_fields_dict
+
+    def get_custom_fields(self, card_id: str) -> objects.CardCustomFields:
+        card_fields = objects.CardCustomFields(card_id)
+        card_fields_dict = self.get_board_custom_fields_dict(card_id)
+        card_fields._data = card_fields_dict
+        logger.info(card_fields)
+
     def get_members(self, board_id) -> List[objects.TrelloMember]:
         _, data = self._make_request(f"api/v2/boards/{board_id}/members")
         members = []
@@ -94,12 +152,13 @@ def get_members(self, board_id) -> List[objects.TrelloMember]:
         logger.debug(f"get_members: {members}")
         return members
 
-    def get_cards(self, list_ids, board_id):
+    def get_cards(self, list_ids):
+        board_id = self.board_id
         _, data = self._make_request(f"api/v2/boards/{board_id}/blocks?all=true")
         cards = []
         # TODO: move this to app state
         members = self.get_members(board_id)
-        lists = self.get_lists(board_id)
+        lists = self.get_lists()
         list_prop = self._get_list_property(board_id)
         member_prop = self._get_member_property(board_id)
         view_id = [card_dict for card_dict in data if card_dict["type"] == "view"][0][
@@ -146,12 +205,22 @@ def _update_from_config(self):
         """Update attributes according to current self._focalboard_config"""
         self.token = self._focalboard_config["token"]
         self.url = self._focalboard_config["url"]
+        self.board_id = self._focalboard_config["board_id"]
         self.headers = {
             "Accept": "application/json",
             "Content-Type": "application/json",
             "Authorization": f"Bearer {self.token}",
             "X-Requested-With": "XMLHttpRequest",
         }
+        lists = self.get_lists()
+        self.lists_config = self._fill_alias_id_map(lists, TrelloListAlias)
+        custom_field_types = self.get_board_custom_field_types()
+        self.custom_fields_type_config = self._fill_id_type_map(
+            custom_field_types, TrelloCustomFieldTypes
+        )
+        self.custom_fields_config = self._fill_alias_id_map(
+            custom_field_types, TrelloCustomFieldTypeAlias
+        )
 
     def _make_request(self, uri, payload={}):
         response = requests.get(

src/jobs/__init__.py

@@ -17,6 +17,7 @@
 from .editorial_board_visual_stats_job import EditorialBoardVisualStatsJob
 from .editorial_report_job import EditorialReportJob
 from .fb_analytics_report_job import FBAnalyticsReportJob
+from .fill_posts_list_focalboard_job import FillPostsListFocalboardJob
 from .fill_posts_list_job import FillPostsListJob
 from .hr_acquisition_job import HRAcquisitionJob
 from .hr_acquisition_pt_job import HRAcquisitionPTJob

src/jobs/fill_posts_list_focalboard_job.py

@@ -0,0 +1,104 @@
+import datetime
+import logging
+import time
+from typing import Callable, List
+
+from ..app_context import AppContext
+from ..consts import TrelloCardColor, TrelloListAlias
+from ..focalboard.focalboard_client import FocalboardClient
+from ..sheets.sheets_objects import RegistryPost
+from ..strings import load
+from ..tg.sender import pretty_send
+from ..trello.trello_client import TrelloClient
+from .base_job import BaseJob
+from .utils import check_trello_card, format_errors
+
+logger = logging.getLogger(__name__)
+
+
+class FillPostsListFocalboardJob(BaseJob):
+    @staticmethod
+    def _execute(
+        app_context: AppContext,
+        send: Callable[[str], None],
+        called_from_handler=False,
+    ):
+        errors = {}
+        registry_posts = []
+        all_rubrics = app_context.db_client.get_rubrics()
+
+        registry_posts += FillPostsListFocalboardJob._retrieve_cards_for_registry(
+            focalboard_client=app_context.focalboard_client,
+            trello_client=app_context.trello_client,
+            list_aliases=(TrelloListAlias.PROOFREADING, TrelloListAlias.DONE),
+            all_rubrics=all_rubrics,
+            errors=errors,
+            show_due=True,
+            strict_archive_rules=True,
+        )
+
+        if len(errors) == 0:
+            posts_added = app_context.sheets_client.update_posts_registry(
+                registry_posts
+            )
+            if len(posts_added) == 0:
+                paragraphs = [load("fill_posts_list_job__unchanged")]
+            else:
+                paragraphs = [load("fill_posts_list_job__success")] + [
+                    "\n".join(
+                        f"{index + 1}) {post_name}"
+                        for index, post_name in enumerate(posts_added)
+                    )
+                ]
+        else:
+            paragraphs = format_errors(errors)
+
+        pretty_send(paragraphs, send)
+
+    @staticmethod
+    def _retrieve_cards_for_registry(
+        focalboard_client: FocalboardClient,
+        trello_client: TrelloClient,
+        list_aliases: List[str],
+        errors: dict,
+        all_rubrics: List,
+        show_due=True,
+        need_illustrators=True,
+        strict_archive_rules=True,
+    ) -> List[str]:
+        """
+        Returns a list of paragraphs that should always go in a single message.
+        """
+        list_ids = focalboard_client.get_list_id_from_aliases(list_aliases)
+        cards = focalboard_client.get_cards(list_ids)
+        if show_due:
+            cards.sort(key=lambda card: card.due or datetime.datetime.min)
+        parse_failure_counter = 0
+
+        registry_posts = []
+
+        for card in cards:
+            # label_names = [label.name for label in card.labels]
+            # is_main_post = load("common_trello_label__main_post") in label_names
+            # is_archive_post = load("common_trello_label__archive") in label_names
+
+            if not card:
+                parse_failure_counter += 1
+                continue
+
+            card_fields = focalboard_client.get_custom_fields(card.id)
+            logger.info(card_fields)
+
+            # registry_posts.append(
+            #     RegistryPost(
+            #         card,
+            #         card_fields,
+            #         is_main_post,
+            #         is_archive_post,
+            #         all_rubrics,
+            #     )
+            # )
+
+        if parse_failure_counter > 0:
+            logger.error(f"Unparsed cards encountered: {parse_failure_counter}")
+        return registry_posts

src/trello/trello_objects.py

@@ -291,6 +291,22 @@ def from_dict(cls, data):
             logger.error(f"Bad field type json {data}: {e}")
         return field_type
 
+    @classmethod
+    def from_focalboard_dict(cls, data):
+        field_type = cls()
+        try:
+            field_type.id = data["id"]
+            field_type.name = html.escape(data["name"])
+            field_type.type = TrelloCustomFieldTypes(data["type"])
+            if field_type.type == TrelloCustomFieldTypes.LIST:
+                field_type.options = {
+                    option["id"]: option["value"] for option in data["options"]
+                }
+        except Exception as e:
+            field_type._ok = False
+            logger.error(f"Bad field type json {data}: {e}")
+        return field_type
+
     def to_dict(self):
         dct = {
             "id": self.id,