feat: Add allocation generation step and preparing reviews for import…

… | NPG-7896 (#521)

# Description

- Added allocations generation step to review importer tool.
- Added preparing reviews step to review importer tool.
- Update config loading, load configs from the event-db.

src/event-db/stage_data/dev/00002_fund10_ideascale_params.sql

@@ -4,6 +4,9 @@ INSERT INTO config (id, id2, id3, value) VALUES (
     '10',
     '',
     '{  
+        "group_id": 31051,
+        "review_stage_ids": [139],
+        "nr_allocations": [30, 80],
         "campaign_group_id": 63,
         "stage_ids": [4590, 4596, 4602, 4608, 4614, 4620, 4626, 4632, 4638, 4644, 4650, 4656, 4662, 4591, 4597, 4603, 4609, 4615, 4621, 4627, 4633, 4639, 4645, 4651, 4657, 4663, 4592, 4598, 4604, 4610, 4616, 4622, 4628, 4634, 4640, 4646, 4652, 4658, 4664],
         "proposals": {

src/event-db/stage_data/testnet/00002_fund10_ideascale_params.sql

@@ -4,6 +4,9 @@ INSERT INTO config (id, id2, id3, value) VALUES (
     '10',
     '',
     '{
+        "group_id": 31051,
+        "review_stage_ids": [139],
+        "nr_allocations": [30, 80],
         "campaign_group_id": 63,
         "stage_ids": [4590, 4596, 4602, 4608, 4614, 4620, 4626, 4632, 4638, 4644, 4650, 4656, 4662, 4591, 4597, 4603, 4609, 4615, 4621, 4627, 4633, 4639, 4645, 4651, 4657, 4663, 4592, 4598, 4604, 4610, 4616, 4622, 4628, 4634, 4640, 4646, 4652, 4658, 4664],
         "proposals": {

utilities/ideascale-importer/ideascale_importer/cli/reviews.py

@@ -1,18 +1,18 @@
 import asyncio
 import typer
-
+import traceback
 from typing import List
+from loguru import logger
 
-from ideascale_importer.reviews_importer.importer import Importer, FrontendClient
+from ideascale_importer.reviews_importer.importer import Importer
 from ideascale_importer.utils import configure_logger
-from loguru import logger
 
 app = typer.Typer(add_completion=False)
 
 @app.command(name="import")
 def import_reviews(
     ideascale_url: str = typer.Option(
-        FrontendClient.DEFAULT_API_URL,
+        ...,
         envvar="IDEASCALE_API_URL",
         help="IdeaScale API URL",
     ),
@@ -29,22 +29,22 @@ def import_reviews(
         ...,
         help="Ideascale user's password (needs admin access)",
     ),
+    event_id: int = typer.Option(
+        ...,
+        help="Database row id of the event which data will be imported",
+    ),
     api_token: str = typer.Option(...,
         envvar="IDEASCALE_API_TOKEN",
         help="IdeaScale API token"
     ),
-    funnel_id: int = typer.Option(
-        ...,
-        help="Ideascale campaign funnel's id",
-    ),
-    nr_allocations: List[int] = typer.Option(
-        [30, 80], 
-        help="Nr of proposal to allocate"
-    ),
     pa_path: str = typer.Option(
-        ...,
+        ..., 
         help="PAs file"
     ),
+    output_path: str = typer.Option(
+        ...,
+        help="output path"
+    ),
     log_level: str = typer.Option(
         "info",
         envvar="REVIEWS_LOG_LEVEL",
@@ -61,21 +61,22 @@ def import_reviews(
 
     async def inner():
         importer = Importer(
-            ideascale_url,
-            database_url,
-            email,
-            password,
-            api_token,
-            funnel_id,
-            nr_allocations,
-            pa_path
+            ideascale_url=ideascale_url,
+            database_url=database_url,
+            email=email,
+            password=password,
+            api_token=api_token,
+            event_id=event_id,
+            pa_path=pa_path,
+            output_path=output_path
         )
 
         try:
             await importer.connect()
             await importer.run()
             await importer.close()
         except Exception as e:
+            traceback.print_exc()
             logger.error(e)
 
     asyncio.run(inner())

utilities/ideascale-importer/ideascale_importer/ideascale/importer.py

@@ -57,14 +57,6 @@ def from_json(val: dict):
         """Load configuration from a JSON object."""
         return pydantic.tools.parse_obj_as(Config, val)
 
-
-class ReadConfigException(Exception):
-    """Raised when the configuration file cannot be read."""
-
-    def __init__(self, cause: str):
-        super().__init__(f"Failed to read config file: {cause}")
-
-
 class ReadProposalsScoresCsv(Exception):
     """Raised when the proposals impact scores csv cannot be read."""
 

utilities/ideascale-importer/ideascale_importer/reviews_importer/importer.py

@@ -1,17 +1,19 @@
 from lxml import html
-import secrets
 import time
 from loguru import logger
+from dataclasses import dataclass
+from typing import List
+import pydantic
+import tempfile
 
 from ideascale_importer import utils
 import ideascale_importer.db
+from .processing.prepare import allocate, process_ideascale_reviews
 
 
 class FrontendClient:
     """IdeaScale front-end client."""
 
-    DEFAULT_API_URL = "https://cardano.ideascale.com"
-
     def __init__(self, ideascale_url):
         self.inner = utils.HttpClient(ideascale_url)
 
@@ -30,12 +32,12 @@ async def login(self,  email, password):
 
         await self.inner.post(f"{login}", data=data)
 
-    async def download_reviews(self, funnel_id):
-        async def download_file(self, funnel_id, id):
+    async def download_reviews(self, reviews_path, review_stage_ids):
+        async def download_file(self, review_stage_id):
             export_endpoint = "/a/admin/workflow/survey-tools/assessment/report/statistic/export/assessment-details/"
-            file_name = f"{funnel_id}_{secrets.token_hex(6)}"
+            file_name = f"{reviews_path}/{review_stage_id}.xlsx"
 
-            content = await self.inner.get(f"{export_endpoint}{id}") 
+            content = await self.inner.get(f"{export_endpoint}{review_stage_id}") 
             tree = html.fromstring(content)
 
             # we are looking for '<div class="card panel export-result-progress" data-features="refresh-processing-item" data-processing-item-id="15622">'
@@ -50,48 +52,56 @@ async def download_file(self, funnel_id, id):
                     download_endpoint = "/a/download-export-file/"
 
                     content = await self.inner.get(f"{download_endpoint}{item}")
-                    return content
-
-
-        funnel_endpoint = "/a/admin/workflow/stages/funnel/"
-        content = await self.inner.get(f"{funnel_endpoint}{funnel_id}")
-
-        # we are looking for '<a href="/a/admin/workflow/survey-tools/assessment/report/statistic/139?fromStage=1">Assessments</a>'
-        # where we need to get url
-        tree = html.fromstring(content)
-        items = tree.findall('.//a')
+                    f = open(file_name, "wb")
+                    f.write(content)
+                    return file_name
         files = []
-        for item in items:
-            if item.text and "Assessments" in item.text:
-                id = int(item.get("href").replace("/a/admin/workflow/survey-tools/assessment/report/reviews/", "").split("?")[0])
-
-                # we are intrested in only assessed reviews 
-                files.append(await download_file(self, funnel_id, id))
+        for review_stage_id in review_stage_ids:
+            # we are interested in only assessed reviews 
+            files.append(await download_file(self, review_stage_id))
         return files
-
 class Importer:
     def __init__(
         self,
         ideascale_url,
         database_url,
         email,
         password,
+        event_id,
         api_token,
-        funnel_id,
-        nr_allocations,
         pa_path,
+        output_path,
     ):
         self.ideascale_url = ideascale_url
         self.database_url = database_url
         self.email = email
         self.password = password
+        self.event_id = event_id
         self.api_token = api_token
-        self.funnel_id = funnel_id
-        self.nr_allocations =  {i: el for i, el in enumerate(nr_allocations)}
+
+        self.pa_path = pa_path
+        self.output_path = output_path
+
+        self.reviews_dir = tempfile.TemporaryDirectory()
+        self.allocations_dir = tempfile.TemporaryDirectory()
 
         self.frontend_client = None
         self.db = None
 
+    async def load_config(self):
+        """Load the configuration setting from the event db."""
+
+        logger.info("Loading ideascale config from the event-db")
+
+        config = ideascale_importer.db.models.Config(row_id=0, id="ideascale", id2=f"{self.event_id}", id3="", value=None)
+        res = await ideascale_importer.db.select(self.db, config,  cond={
+            "id": f"= '{config.id}'",
+            "AND id2": f"= '{config.id2}'"
+            })
+        if len(res) == 0:
+            raise Exception("Cannot find ideascale config in the event-db database")
+        self.config = Config.from_json(res[0].value)
+
     async def connect(self):
         if self.frontend_client is None:
             logger.info("Connecting to the Ideascale frontend")
@@ -103,20 +113,68 @@ async def connect(self):
 
     async def download_reviews(self):
         logger.info("Dowload reviews from Ideascale...")
-        await self.frontend_client.download_reviews(self.funnel_id)
+
+        self.reviews = await self.frontend_client.download_reviews(self.reviews_dir.name, self.config.review_stage_ids)
 
     async def prepare_allocations(self):
         logger.info("Prepare allocations for proposal's reviews...")
 
+        self.allocations_path = await allocate(
+            nr_allocations=self.config.nr_allocations,
+            pas_path=self.pa_path,
+            ideascale_api_key=self.api_token,
+            ideascale_api_url=self.ideascale_url,
+            stage_ids=self.config.stage_ids,
+            challenges_group_id=self.config.campaign_group_id,
+            group_id=self.config.group_id,
+            output_path=self.allocations_dir.name,
+        )
+
+    async def prepare_reviews(self):
+        logger.info("Prepare proposal's reviews...")
+
+        for review in self.reviews:
+            await process_ideascale_reviews(
+                ideascale_xlsx_path=review,
+                ideascale_api_url=self.ideascale_url,
+                ideascale_api_key=self.api_token,
+                allocation_path=self.allocations_path,
+                challenges_group_id=self.config.campaign_group_id,
+                fund=self.event_id,
+                output_path=self.output_path
+            )
+
+    async def import_reviews(self):
+        logger.info("Import reviews into Event db")
 
     async def run(self):
         """Run the importer."""
         if self.frontend_client is None:
             raise Exception("Not connected to the ideascale")
 
-        # await self.download_reviews()
+        await self.load_config()
+
+        await self.download_reviews()
         await self.prepare_allocations()
+        await self.prepare_reviews()
 
     async def close(self):
+        self.reviews_dir.cleanup()
+        self.allocations_dir.cleanup()
         await self.frontend_client.close()
 
+@dataclass
+class Config:
+    """Represents the available configuration fields."""
+
+    group_id: int
+    campaign_group_id: int
+    review_stage_ids: List[int]
+    stage_ids: List[int]
+    nr_allocations: List[int]
+
+    @staticmethod
+    def from_json(val: dict):
+        """Load configuration from a JSON object."""
+        return pydantic.tools.parse_obj_as(Config, val)
+