Merge pull request #1 from mapswipe/refactor-python-workers

Refactor python workers

Author: ElJocho

docker-compose.development.yaml

@@ -25,7 +25,7 @@ services:
             - manager_dashboard
 
     postgres:
-        container_name: postgres_mapswipe_workers
+        container_name: postgres
         build:
             context: postgres/
             dockerfile: Dockerfile-dev
@@ -39,6 +39,6 @@ services:
           - ./postgres-data:/var/lib/postgresql/mapswipe
         restart: unless-stopped
         ports:
-          - "5431:5432"
+          - "${POSTGRES_PORT}:5432"
         networks:
             - postgres

mapswipe_workers/.pre-commit-config.yaml

@@ -4,11 +4,11 @@ repos:
     rev: 22.3.0
     hooks:
       - id: black
-  - repo: https://gitlab.com/pycqa/flake8
+  - repo: https://github.com/PyCQA/flake8/
     rev: 3.8.3
     hooks:
       - id: flake8
   - repo: https://github.com/PyCQA/isort
-    rev: 5.5.2
+    rev: 5.12.0
     hooks:
       - id: isort

mapswipe_workers/mapswipe_workers/config.py

@@ -1,5 +1,9 @@
 import os
 
+from dotenv import load_dotenv
+
+load_dotenv()
+
 FIREBASE_API_KEY = os.environ["FIREBASE_API_KEY"]
 FIREBASE_DB = os.getenv("FIREBASE_DB", default="mapswipe")
 

mapswipe_workers/mapswipe_workers/project_types/tile_classification/project.py

@@ -1,9 +1,10 @@
 from mapswipe_workers.firebase.firebase import Firebase
 from mapswipe_workers.project_types.base.project import BaseProject
-
 from mapswipe_workers.project_types.base.tile_server import BaseTileServer
-
-from mapswipe_workers.utils.validate_input import validate_geometries
+from mapswipe_workers.utils.validate_input import (
+    save_geojson_to_file,
+    validate_geometries,
+)
 
 
 class TileClassification(BaseProject):
@@ -15,9 +16,15 @@ def __init__(self, project_draft: dict):
         self.tileServer = vars(BaseTileServer(project_draft["tileServer"]))
 
     def validate_geometries(self):
-        wkt_geometry, self.validInputGeometries = validate_geometries(self.projectId, self.geometry, self.zoomLevel)
+        self.validInputGeometries = save_geojson_to_file(self.projectId, self.geometry)
+        wkt_geometry, self.validInputGeometries = validate_geometries(
+            self.projectId, self.validInputGeometries, self.zoomLevel
+        )
         return wkt_geometry
 
+    def create_groups(self):
+        pass
+     
     def save_project_to_firebase(self, project):
         firebase = Firebase()
         firebase.save_project_to_firebase(project)
@@ -27,4 +34,6 @@ def save_groups_to_firebase(self, projectId: str, groups: list):
         firebase.save_groups_to_firebase(projectId, groups)
 
     def save_tasks_to_firebase(self, projectId: str, tasks: list):
-        pass
+        pass
+
+      

mapswipe_workers/mapswipe_workers/utils/validate_input.py

@@ -1,24 +1,33 @@
 import json
 import os
 
-from mapswipe_workers.definitions import CustomError, logger, MAX_INPUT_GEOMETRIES, DATA_PATH
 from osgeo import ogr, osr
 
+from mapswipe_workers.definitions import (
+    DATA_PATH,
+    MAX_INPUT_GEOMETRIES,
+    CustomError,
+    logger,
+)
 
-def validate_geometries(projectId, geometry, zoomLevel):
-    raw_input_file = (
-        f"{DATA_PATH}/input_geometries/" f"raw_input_{projectId}.geojson"
+
+def save_geojson_to_file(project_id, geometry):
+    output_file_path = (
+        f"{DATA_PATH}/input_geometries/" f"raw_input_{project_id}.geojson"
     )
     # check if a 'data' folder exists and create one if not
     if not os.path.isdir("{}/input_geometries".format(DATA_PATH)):
         os.mkdir("{}/input_geometries".format(DATA_PATH))
 
     # write string to geom file
-    with open(raw_input_file, "w") as geom_file:
+    with open(output_file_path, "w") as geom_file:
         json.dump(geometry, geom_file)
+    return output_file_path
+
 
+def validate_geometries(projectId, zoomLevel, input_file_path):
     driver = ogr.GetDriverByName("GeoJSON")
-    datasource = driver.Open(raw_input_file, 0)
+    datasource = driver.Open(input_file_path, 0)
 
     try:
         layer = datasource.GetLayer()
@@ -147,11 +156,9 @@ def validate_geometries(projectId, geometry, zoomLevel):
     del datasource
     del layer
 
-    logger.info(
-        f"{projectId}" f" - validate geometry - " f"input geometry is correct."
-    )
+    logger.info(f"{projectId}" f" - validate geometry - " f"input geometry is correct.")
 
     dissolved_geometry = geometry_collection.UnionCascaded()
     wkt_geometry_collection = dissolved_geometry.ExportToWkt()
 
-    return wkt_geometry_collection, raw_input_file
+    return wkt_geometry_collection

mapswipe_workers/requirements.txt

@@ -13,3 +13,4 @@ sentry-sdk==0.18.0
 six==1.15.0
 slackclient==2.9.2
 xdg==4.0.1
+python-dotenv==1.0.0

mapswipe_workers/tests/unittests/fixtures/tile_map_service_grid/projectDrafts/tile_classification.json

@@ -0,0 +1,16 @@
+{
+    "createdBy": "test",
+    "geometry": {"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[34.975833892822266,-15.899098066386088],[35.089302062988274,-15.899098066386088],[35.089302062988274,-15.820002241903946],[34.975833892822266,-15.820002241903946],[34.975833892822266,-15.899098066386088]]]},"properties":{}}]},
+    "image": "",
+    "lookFor": "buildings",
+    "name": "test - Malawi (1)\ntest",
+    "projectDetails": "test",
+    "verificationNumber": 3,
+    "groupSize": 120,
+    "tileServer": {
+        "name": "bing",
+        "credits": "© 2019 Microsoft Corporation, Earthstar Geographics SIO"
+    },
+    "projectType": 1,
+    "projectDraftId": "asdfghtjue"
+}

mapswipe_workers/tests/unittests/test_init_project.py

@@ -0,0 +1,22 @@
+import json
+import os
+import unittest
+
+from mapswipe_workers.project_types.tile_classification.project import (
+    TileClassificationProject,
+)
+
+
+class TestInitProject(unittest.TestCase):
+    def test_init_tile_classification_project(self):
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        path = "fixtures/tile_map_service_grid/projectDrafts/tile_classification.json"
+
+        with open(os.path.join(test_dir, path)) as json_file:
+            project_draft = json.load(json_file)
+
+        self.assertIsNotNone(TileClassificationProject(project_draft=project_draft))
+
+
+if __name__ == "__main__":
+    unittest.main()

mapswipe_workers/tests/unittests/test_validate_input.py

@@ -1,40 +1,127 @@
+import json
 import os
 import unittest
-import json
+
+from osgeo import ogr
 
 from mapswipe_workers.definitions import CustomError
-from mapswipe_workers.utils.validate_input import validate_geometries
+from mapswipe_workers.utils.validate_input import (
+    save_geojson_to_file,
+    validate_geometries,
+)
 
 
-class TestValidateGeometries(unittest.TestCase):
-    def test_multiple_geom_validation(self):
-        pass
+def get_project_draft(path):
+    # pre steps for outputting result of function
+    test_dir = os.path.dirname(os.path.abspath(__file__))
+    with open(os.path.join(test_dir, path)) as json_file:
+        project_draft = json.load(json_file)
+
+    path_to_geometries = save_geojson_to_file(1, project_draft["geometry"])
+    return project_draft, path_to_geometries
 
-    # todo: all the other tests
 
+class TestValidateGeometries(unittest.TestCase):
     def test_area_is_too_large(self):
         """Test if validate_geometries throws an error
         if the provided geojson covers a too large area."""
 
         path = (
             "fixtures/tile_map_service_grid/projects/projectDraft_area_too_large.json"
         )
-        test_dir = os.path.dirname(os.path.abspath(__file__))
-        with open(os.path.join(test_dir, path)) as json_file:
-            project_draft = json.load(json_file)
-            geometry = project_draft["geometry"]
+        project_draft, path_to_geometries = get_project_draft(path)
+        self.assertRaises(CustomError, validate_geometries, 1, 18, path_to_geometries)
+
+    def test_broken_geojson_string(self):
+        """Test if validate_geometries throws an error
+        if the provided geojson string is broken.
+        This means we can't create the geo data layer with ogr."""
+
+        path = (
+            "fixtures/tile_map_service_grid/projects/projectDraft_broken_geojson.json"
+        )
+        project_draft, path_to_geometries = get_project_draft(path)
+        self.assertRaises(CustomError, validate_geometries, 1, 18, path_to_geometries)
+
+    def test_feature_is_none(self):
+        """Test if validate_geometries throws an error
+        if the provided geojson contains a not defined feature."""
+
+        path = (
+            "fixtures/tile_map_service_grid/projects/projectDraft_feature_is_none.json"
+        )
+        project_draft, path_to_geometries = get_project_draft(path)
+        self.assertRaises(CustomError, validate_geometries, 1, 18, path_to_geometries)
+
+    def test_no_features(self):
+        """Test if validate_geometries throws an error
+        if the provided geojson contains no features."""
+
+        path = "fixtures/tile_map_service_grid/projects/projectDraft_no_features.json"
+        project_draft, path_to_geometries = get_project_draft(path)
+        self.assertRaises(CustomError, validate_geometries, 1, 18, path_to_geometries)
+
+    def test_single_geom_validation(self):
+        path = "fixtures/completeness/projectDraft_single.json"
+        project_draft, path_to_geometries = get_project_draft(path)
+
+        path = "fixtures/completeness/single_polygon.geojson"
+        driver = ogr.GetDriverByName("GeoJSON")
+        datasource = driver.Open(path, 0)
+
+        geometry_collection = ogr.Geometry(ogr.wkbMultiPolygon)
+        # Get the data layer
+        layer = datasource.GetLayer()
+        for feature in layer:
+            feat_geom = feature.GetGeometryRef()
+            geom_name = feat_geom.GetGeometryName()
+            # add geometry to geometry collection
+            if geom_name == "MULTIPOLYGON":
+                for multi in feat_geom:
+                    geometry_collection.AddGeometry(multi)
+            if geom_name == "POLYGON":
+                geometry_collection.AddGeometry(feat_geom)
+
+        dissolved_geometry = geometry_collection.UnionCascaded()
+        wkt_geometry_collection = dissolved_geometry.ExportToWkt()
+
+        # results coming from the validate_geometries function
+        wkt = validate_geometries(1, 18, path_to_geometries)
+        # Test that sequence first contains the same elements as second
+        self.assertCountEqual(wkt, wkt_geometry_collection)
+
+    def test_multiple_geom_validation(self):
+        path = "fixtures/completeness/projectDraft_overlappingGeom.json"
+        project_draft, path_to_geometries = get_project_draft(path)
 
-        self.assertRaises(CustomError, validate_geometries, 1, geometry, 18)
+        # prepare data that is expected
+        path = "fixtures/completeness/overlappingGeoms.geojson"
+        driver = ogr.GetDriverByName("GeoJSON")
+        datasource = driver.Open(path, 0)
 
-        """
-        project = create_project(path)
+        geometry_collection = ogr.Geometry(ogr.wkbMultiPolygon)
+        # Get the data layer
+        layer = datasource.GetLayer()
+        for feature in layer:
+            feat_geom = feature.GetGeometryRef()
+            geom_name = feat_geom.GetGeometryName()
+            # add geometry to geometry collection
+            # check if input geom is multipolygon or polygon
+            if geom_name == "MULTIPOLYGON":
+                for multi in feat_geom:
+                    geometry_collection.AddGeometry(multi)
+                # apply union function if multiple geoms of polygons overlap
+                dissolved_geometry = geometry_collection.UnionCascaded()
+                wkt_geometry_collection = dissolved_geometry.ExportToWkt()
+            if geom_name == "POLYGON":
+                geometry_collection.AddGeometry(feat_geom)
+                wkt_geometry_collection = geometry_collection.ExportToWkt()
 
+        # results coming from the validate_geometries function
+        wkt = validate_geometries(1, 18, path_to_geometries)
+        # Test that sequence first contains the same elements as second
+        self.assertEqual(wkt, wkt_geometry_collection)
 
-        
-        # we expect that the function raises a CustomError due to the fact
-        # that the area is too large
-        self.assertRaises(CustomError, project.validate_geometries)
-"""
 
 if __name__ == "__main__":
     unittest.main()