fea(re)

Author: ElJocho

mapswipe_workers/mapswipe_workers/firebase/firebase.py

@@ -0,0 +1,61 @@
+from mapswipe_workers import auth
+from mapswipe_workers.definitions import logger
+from mapswipe_workers.utils import gzip_str
+
+
+class Firebase:
+    def __init__(self):
+        self.fb_db = auth.firebaseDB()
+        self.ref = self.fb_db.reference("")
+
+    def save_project_to_firebase(self, project):
+        # if a geometry exists in projects we want to delete it since it is not used in clients.
+        project.pop("geometry", None)
+        # save project
+        self.ref.update({f"v2/projects/{project['projectId']}": project})
+        logger.info(
+            f"{project['projectId']} -" f" uploaded project to firebase realtime database"
+        )
+
+
+    def save_groups_to_firebase(self, projectId, groups):
+
+        # save groups
+        self.ref.update({f"v2/groups/{projectId}": groups})
+        logger.info(
+            f"{projectId} -" f" uploaded groups to firebase realtime database"
+        )
+
+    def save_tasks_to_firebase(self, projectId, groupsOfTasks, useCompression: bool):
+        task_upload_dict = {}
+        for group_counter, group_id in enumerate(groupsOfTasks.keys()):
+            for i in range(0, len(groupsOfTasks[group_id])):
+                groupsOfTasks[group_id][i].pop("geometry", None)
+
+            tasks_list = groupsOfTasks[group_id]
+            # for tasks of a building footprint project
+            # we use compression to reduce storage size in firebase
+            # since the tasks hold geometries their storage size
+            # can get quite big otherwise
+            if useCompression:
+                # removing properties from each task and compress
+                for task in tasks_list:
+                    task.pop("properties", None)
+
+                tasks_list = gzip_str.compress_tasks(tasks_list)
+
+            task_upload_dict[
+                f"v2/tasks/{projectId}/{group_id}"
+            ] = tasks_list
+
+            # we upload tasks in batches of maximum 150 groups
+            # this is to avoid the maximum write size limit in firebase
+            if len(task_upload_dict) % 150 == 0 or (group_counter + 1) == len(
+                groupsOfTasks
+            ):
+                self.ref.update(task_upload_dict)
+                logger.info(
+                    f"{projectId} -"
+                    f" uploaded 150 groups with tasks to firebase realtime database"
+                )
+                task_upload_dict = {}

mapswipe_workers/mapswipe_workers/project_types/base/project.py

@@ -172,11 +172,15 @@ def save_project(self):
             raise CustomError(e)
 
         try:
-            self.save_to_firebase(
+            self.save_project_to_firebase(project)
+            self.save_groups_to_firebase(project["projectId"], groups)
+            self.save_tasks_to_firebase(project["projectId"], groupsOfTasks)
+
+            """self.save_to_firebase(
                 project,
                 groups,
                 groupsOfTasks,
-            )
+            )"""
             logger.info(
                 f"{self.projectId}" f" - the project has been saved" f" to firebase"
             )
@@ -197,6 +201,18 @@ def save_project(self):
 
         return True
 
+    @abstractmethod
+    def save_project_to_firebase(self, project):
+        pass
+
+    @abstractmethod
+    def save_groups_to_firebase(self, projectId: str, groups: list):
+        pass
+
+    @abstractmethod
+    def save_tasks_to_firebase(self, projectId: str, tasks: list):
+        pass
+
     def save_to_firebase(self, project, groups, groupsOfTasks):
 
         # remove wkt geometry attribute of projects and tasks

mapswipe_workers/mapswipe_workers/project_types/tile_classification/project.py

@@ -1,3 +1,4 @@
+from mapswipe_workers.firebase.firebase import Firebase
 from mapswipe_workers.project_types.base.project import BaseProject
 
 from mapswipe_workers.project_types.base.tile_server import BaseTileServer
@@ -6,7 +7,6 @@
 
 
 class TileClassification(BaseProject):
-    #TileServer tileServer
     def __init__(self, project_draft: dict):
         super().__init__(project_draft)
         # Note: this will be overwritten by validate_geometry in mapswipe_workers.py
@@ -17,3 +17,14 @@ def __init__(self, project_draft: dict):
     def validate_geometries(self):
         wkt_geometry, self.validInputGeometries = validate_geometries(self.projectId, self.geometry, self.zoomLevel)
         return wkt_geometry
+
+    def save_project_to_firebase(self, project):
+        firebase = Firebase()
+        firebase.save_project_to_firebase(project)
+
+    def save_groups_to_firebase(self, projectId: str, groups: list):
+        firebase = Firebase()
+        firebase.save_groups_to_firebase(projectId, groups)
+
+    def save_tasks_to_firebase(self, projectId: str, tasks: list):
+        pass

mapswipe_workers/tests/integration/fixtures/tile_map_service_grid/projects/build_area_with_geometry.json

@@ -0,0 +1,31 @@
+{
+  "geometry": {"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[34.975833892822266,-15.899098066386088],[35.089302062988274,-15.899098066386088],[35.089302062988274,-15.820002241903946],[34.975833892822266,-15.820002241903946],[34.975833892822266,-15.899098066386088]]]},"properties":{}}]},
+  "archived" : false,
+  "created" : "2020-02-03T14:31:08.192136Z",
+  "createdBy" : "test manager",
+  "groupMaxSize" : 0,
+  "groupSize" : 120,
+  "image" : "",
+  "isFeatured" : false,
+  "lookFor" : "buildings",
+  "name" : "test - Malawi \ntest",
+  "progress" : 0,
+  "projectDetails" : "test",
+  "projectId" : "test_build_area",
+  "projectNumber" : "1",
+  "projectRegion" : "Malawi",
+  "projectTopic" : "test",
+  "projectType" : 1,
+  "requestingOrganisation" : "test",
+  "requiredResults" : 12537,
+  "resultCount" : 0,
+  "status" : "inactive",
+  "tileServer" : {
+    "apiKey" : "",
+    "credits" : "© 2019 Microsoft Corporation, Earthstar Geographics SIO",
+    "name" : "bing",
+    "url" : "https://ecn.t0.tiles.virtualearth.net/tiles/a{quad_key}.jpeg?g=7505&mkt=en-US&token={key}"
+  },
+  "verificationNumber" : 3,
+  "zoomLevel" : 18
+}

mapswipe_workers/tests/integration/test_firebase.py

@@ -0,0 +1,85 @@
+import unittest
+import os
+import json
+
+from firebase_admin.exceptions import FirebaseError
+
+from mapswipe_workers import auth
+from mapswipe_workers.firebase.firebase import Firebase
+from tests.integration import tear_down
+
+
+class TestFirebase(unittest.TestCase):
+    def setUp(self):
+        self.firebase = Firebase()
+        self.ids = []
+
+    def tearDown(self):
+        self.fb_db = None
+        for id in self.ids:
+            tear_down.delete_test_data(id)
+
+    def test_project_to_firebase(self):
+        path = (
+            "./fixtures/tile_map_service_grid/projects/build_area_with_geometry.json"
+        )
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        with open(os.path.join(test_dir, path)) as json_file:
+            project = json.load(json_file)
+        self.ids.append(project["projectId"])
+        self.firebase.save_project_to_firebase(project)
+
+        self.firebase.ref = self.firebase.fb_db.reference(f"/v2/projects/{project['projectId']}")
+        result = self.firebase.ref.get(shallow=True)
+        self.assertIsNotNone(result)
+        self.assertNotIn("geometry", result)
+
+    def test_groups_to_firebase(self):
+        path = (
+            "./fixtures/tile_map_service_grid/groups/build_area.json"
+        )
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        with open(os.path.join(test_dir, path)) as json_file:
+            groups = json.load(json_file)
+        project_id = groups["g101"]["projectId"]
+        self.ids.append(project_id)
+        self.firebase.save_groups_to_firebase(project_id, groups)
+
+        self.firebase.ref = self.firebase.fb_db.reference(f"/v2/groups/{project_id}")
+        result = self.firebase.ref.get(shallow=True)
+        self.assertIsNotNone(result)
+
+    def test_tasks_to_firebase_with_compression(self):
+        path = (
+            "./fixtures/tile_map_service_grid/tasks/build_area.json"
+        )
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        with open(os.path.join(test_dir, path)) as json_file:
+            tasks = json.load(json_file)
+        project_id = tasks["g101"][0]["projectId"]
+        self.ids.append(project_id)
+        self.firebase.save_tasks_to_firebase(project_id, tasks, useCompression=True)
+        self.firebase.ref = self.firebase.fb_db.reference(f"/v2/tasks/{project_id}")
+        result = self.firebase.ref.get()
+        self.assertIsNotNone(result)
+        self.assertNotIsInstance(result["g101"], list)
+
+
+    def test_tasks_to_firebase_without_compression(self):
+        path = (
+            "./fixtures/tile_map_service_grid/tasks/build_area.json"
+        )
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        with open(os.path.join(test_dir, path)) as json_file:
+            tasks = json.load(json_file)
+        project_id = tasks["g101"][0]["projectId"]
+        self.ids.append(project_id)
+        self.firebase.save_tasks_to_firebase(project_id, tasks, useCompression=False)
+        self.firebase.ref = self.firebase.fb_db.reference(f"/v2/tasks/{project_id}")
+        result = self.firebase.ref.get()
+        self.assertIsNotNone(result)
+        self.assertIsInstance(result["g101"], list)
+
+
+if __name__ == "__main__":
+    unittest.main()