feat: Move review's processing code from catalyst-export repo | NPG-7…

…896 (#518)

# Description

Move review's processing code from catalyst-export repo. 
It is just a simple copy-past of the python modules, no actuall
processing at this moment has been added into import implementation.
Initially I have tried to smartly move this code to avoid duplication of
some parts, but unfortunatelly it does not seems achievable at least in
reasonable amount of time.

utilities/ideascale-importer/ideascale_importer/cli/reviews.py

@@ -1,8 +1,9 @@
 import asyncio
 import typer
 
-from ideascale_importer.reviews_importer import Client
-from ideascale_importer.reviews_importer import Importer
+from typing import List
+
+from ideascale_importer.reviews_importer.importer import Importer, FrontendClient
 from ideascale_importer.utils import configure_logger
 from loguru import logger
 
@@ -11,10 +12,15 @@
 @app.command(name="import")
 def import_reviews(
     ideascale_url: str = typer.Option(
-        Client.DEFAULT_API_URL,
+        FrontendClient.DEFAULT_API_URL,
         envvar="IDEASCALE_API_URL",
         help="IdeaScale API URL",
     ),
+    database_url: str = typer.Option(
+        ...,
+        envvar="EVENTDB_URL",
+        help="Postgres database URL"
+    ),
     email: str = typer.Option(
         ...,
         help="Ideascale user's email address (needs admin access)",
@@ -23,13 +29,21 @@ def import_reviews(
         ...,
         help="Ideascale user's password (needs admin access)",
     ),
+    api_token: str = typer.Option(...,
+        envvar="IDEASCALE_API_TOKEN",
+        help="IdeaScale API token"
+    ),
     funnel_id: int = typer.Option(
         ...,
         help="Ideascale campaign funnel's id",
     ),
-    out_dir: str = typer.Option(
+    nr_allocations: List[int] = typer.Option(
+        [30, 80], 
+        help="Nr of proposal to allocate"
+    ),
+    pa_path: str = typer.Option(
         ...,
-        help="Output directoy for storing review's files",
+        help="PAs file"
     ),
     log_level: str = typer.Option(
         "info",
@@ -48,10 +62,13 @@ def import_reviews(
     async def inner():
         importer = Importer(
             ideascale_url,
+            database_url,
             email,
             password,
+            api_token,
             funnel_id,
-            out_dir,
+            nr_allocations,
+            pa_path
         )
 
         try:

utilities/ideascale-importer/ideascale_importer/db/models.py

@@ -1,11 +1,11 @@
 """Database models for the vit-servicing-station database."""
 
-import dataclasses
+from dataclasses import dataclass
 from datetime import datetime
 from typing import Any, ClassVar, List, Mapping, Optional, Set
 
 
-@dataclasses.dataclass
+@dataclass
 class Model:
     """Base class for all models."""
 
@@ -17,7 +17,7 @@ def table() -> str:
         raise NotImplementedError()
 
 
-@dataclasses.dataclass
+@dataclass
 class Event(Model):
     """Represents a database objective."""
 
@@ -46,7 +46,7 @@ def table() -> str:
         return "event"
 
 
-@dataclasses.dataclass
+@dataclass
 class Objective(Model):
     """Represents a database objective."""
 
@@ -71,7 +71,7 @@ def table() -> str:
         return "objective"
 
 
-@dataclasses.dataclass
+@dataclass
 class Proposal(Model):
     """Represents a database proposal."""
 
@@ -103,7 +103,7 @@ def table() -> str:
         return "proposal"
 
 
-@dataclasses.dataclass
+@dataclass
 class Goal(Model):
     """Represents a database goal."""
 
@@ -117,7 +117,7 @@ def table() -> str:
         return "goal"
 
 
-@dataclasses.dataclass
+@dataclass
 class VotingGroup(Model):
     """Represents a database voting_group."""
 
@@ -131,7 +131,7 @@ def table() -> str:
         return "voting_group"
 
 
-@dataclasses.dataclass
+@dataclass
 class Voteplan(Model):
     """Represents a database voteplan."""
 
@@ -147,7 +147,7 @@ def table() -> str:
         return "voteplan"
 
 
-@dataclasses.dataclass
+@dataclass
 class ProposalVoteplan(Model):
     """Represents a database proposal_voteplan."""
 
@@ -161,7 +161,7 @@ def table() -> str:
         return "proposal_voteplan"
 
 
-@dataclasses.dataclass
+@dataclass
 class Voter(Model):
     """Represents a database voter."""
 
@@ -176,7 +176,7 @@ def table() -> str:
         return "voter"
 
 
-@dataclasses.dataclass
+@dataclass
 class Contribution(Model):
     """Represents a database contribution."""
 
@@ -195,7 +195,7 @@ def table() -> str:
         return "contribution"
 
 
-@dataclasses.dataclass
+@dataclass
 class Snapshot(Model):
     """Represents a database snapshot."""
 
@@ -219,7 +219,7 @@ def table() -> str:
         """Return the name of the table that this model is stored in."""
         return "snapshot"
 
-@dataclasses.dataclass
+@dataclass
 class Config(Model):
     """Represents a database config."""
 

utilities/ideascale-importer/ideascale_importer/reviews_importer.py → utilities/ideascale-importer/ideascale_importer/reviews_importer/importer.py

@@ -1,10 +1,14 @@
 from lxml import html
 import secrets
+import time
+from loguru import logger
 
 from ideascale_importer import utils
+import ideascale_importer.db
 
-class Client:
-    """IdeaScale front-end API client."""
+
+class FrontendClient:
+    """IdeaScale front-end client."""
 
     DEFAULT_API_URL = "https://cardano.ideascale.com"
 
@@ -26,8 +30,8 @@ async def login(self,  email, password):
 
         await self.inner.post(f"{login}", data=data)
 
-    async def download_reviews(self, funnel_id, out_dir):
-        async def download_file(self, funnel_id, id, out_dir):
+    async def download_reviews(self, funnel_id):
+        async def download_file(self, funnel_id, id):
             export_endpoint = "/a/admin/workflow/survey-tools/assessment/report/statistic/export/assessment-details/"
             file_name = f"{funnel_id}_{secrets.token_hex(6)}"
 
@@ -40,14 +44,13 @@ async def download_file(self, funnel_id, id, out_dir):
 
             export_data_endpoint = "/a/reporting/export-data/"
             while True:
+                time.sleep(2)
                 content = await self.inner.get(f"{export_data_endpoint}{item}")
                 if "Finished Processing" in str(content):
                     download_endpoint = "/a/download-export-file/"
 
                     content = await self.inner.get(f"{download_endpoint}{item}")
-                    f = open(f"{out_dir}/{file_name}.xls", "wb")
-                    f.write(content)
-                    return file_name
+                    return content
 
 
         funnel_endpoint = "/a/admin/workflow/stages/funnel/"
@@ -63,34 +66,57 @@ async def download_file(self, funnel_id, id, out_dir):
                 id = int(item.get("href").replace("/a/admin/workflow/survey-tools/assessment/report/reviews/", "").split("?")[0])
 
                 # we are intrested in only assessed reviews 
-                files.append(await download_file(self, funnel_id, id, out_dir))
+                files.append(await download_file(self, funnel_id, id))
         return files
 
 class Importer:
     def __init__(
         self,
         ideascale_url,
+        database_url,
         email,
         password,
+        api_token,
         funnel_id,
-        out_dir,
+        nr_allocations,
+        pa_path,
     ):
         self.ideascale_url = ideascale_url
+        self.database_url = database_url
         self.email = email
         self.password = password
-        self.out_dir = out_dir
+        self.api_token = api_token
         self.funnel_id = funnel_id
+        self.nr_allocations =  {i: el for i, el in enumerate(nr_allocations)}
+
+        self.frontend_client = None
+        self.db = None
 
     async def connect(self):
-        self.client = Client(self.ideascale_url)
-        await self.client.login(self.email, self.password)
+        if self.frontend_client is None:
+            logger.info("Connecting to the Ideascale frontend")
+            self.frontend_client = FrontendClient(self.ideascale_url)
+            await self.frontend_client.login(self.email, self.password)
+        if self.db is None:
+            logger.info("Connecting to the database")
+            self.db = await ideascale_importer.db.connect(self.database_url)
+
+    async def download_reviews(self):
+        logger.info("Dowload reviews from Ideascale...")
+        await self.frontend_client.download_reviews(self.funnel_id)
+
+    async def prepare_allocations(self):
+        logger.info("Prepare allocations for proposal's reviews...")
+
 
     async def run(self):
         """Run the importer."""
-        if self.client is None:
+        if self.frontend_client is None:
             raise Exception("Not connected to the ideascale")
 
-        await self.client.download_reviews(self.funnel_id, self.out_dir)
+        # await self.download_reviews()
+        await self.prepare_allocations()
 
     async def close(self):
-        await self.client.close()
+        await self.frontend_client.close()
+