Parses and stores attribute data

Closes #10

Author: ridoo

importer_datapackage/handlers/datapackage/data/datapackage.json

@@ -29,11 +29,13 @@
         "fields": [
           {
             "name": "PointID",
+            "title": "Point ID",
             "type": "integer",
             "description": "Sampling point identifier"
           },
           {
             "name": "HorizontID",
+            "title": "Horizont ID",
             "type": "integer",
             "description": "Horizon number of each individual soil profile"
           },
@@ -104,6 +106,7 @@
         "fields": [
           {
             "name": "PointID",
+            "title": "Point ID",
             "description": "Sampling point identifier",
             "type": "integer"
           },
@@ -188,59 +191,69 @@
           },
           {
             "name": "pH_H2O",
+            "title": "pH of H2O",
             "description": "pH-value in water [mol l-1] (DIN ISO 10390, modified)",
             "type": "number",
             "decimalChar": ","
           },
           {
             "name": "EC_H2O",
+            "title": "EC of H2O",
             "description": "Electric conductivity in water [µS cm-1] (DIN EN ISO 27888)",
             "type": "number",
             "decimalChar": ","
           },
           {
             "name": "pH_CaCl2",
+            "title": "pH of CaCl2",
             "description": "pH-vlaue in Claciumchlorid [mol l-1] (DIN ISO 10390)",
             "type": "number",
             "decimalChar": ","
           },
           {
             "name": "TC",
+            "title": "Total Carbon",
             "description": "Total carbon[g kg-1] (DIN ISO 10694:1995)",
             "type": "number",
             "decimalChar": ","
           },
           {
             "name": "TOC",
+            "title": "Organic Carbon",
             "description": "Organic carbon [g kg-1]  (DIN ISO 10694:1995)",
             "type": "number",
             "decimalChar": ","
           },
           {
             "name": "TIC",
+            "title": "Inorganic Carbon",
             "description": "Inorganic carbon [g kg-1] (DIN 19539, modified)",
             "type": "number",
             "decimalChar": ","
           },
           {
             "name": "TN",
+            "title": "Total Nitrogen",
             "description": "Total nitrogen [g kg-1]  (DIN EN 16168:2012)",
             "type": "number",
             "decimalChar": ","
           },
           {
             "name": "TS",
+            "title": "Total Sulphur",
             "description": "Total sulphur [g kg-1] (DIN ISO 15178:02-2001)",
             "type": "string"
           },
           {
             "name": "BD_FS",
+            "title": "Buld Density of Fine Soil",
             "description": "Bulk density of the fine soil [g cm-3] (HFA A2.8, verändert)",
             "type": "number",
             "decimalChar": ","
           },
           {
             "name": "FSS",
+            "title": "Fine Soil Stock",
             "description": "Fine soil stock [Mg ha-1]",
             "type": "number",
             "decimalChar": ","

importer_datapackage/handlers/datapackage/data/valid_data.zip

@@ -18,6 +18,7 @@
 from geonode.resource.manager import resource_manager
 from geonode.resource.models import ExecutionRequest
 from geonode.resource.enumerator import ExecutionRequestAction as exa
+from geonode.geoserver.helpers import set_attributes
 from geonode.utils import set_resource_default_links
 
 from importer.handlers.common.vector import BaseVectorFileHandler
@@ -26,7 +27,7 @@
 
 from frictionless import Package
 
-from .mapper import SchemaToVrtMapper
+from .mapper import TabularDataHelper
 from .util import process_rows, validate
 
 logger = logging.getLogger(__name__)
@@ -99,7 +100,7 @@ def prepare_import(self, files, execution_id, **kwargs):
             process_rows(resource)
 
         folder = Path(_file).parent
-        mapper = SchemaToVrtMapper(package)
+        mapper = TabularDataHelper(package)
         vrt_file = mapper.write_vrt_file(f"{package.name}.vrt", folder)
 
         # update base file to be imported by ogr2ogr
@@ -192,6 +193,13 @@ def create_geonode_resource(
         set_resource_default_links(saved_dataset.get_real_instance(), saved_dataset)
         ResourceBase.objects.filter(alternate=alternate).update(dirty_state=False)
 
+        package_file = files.get("package_file")
+        package = Package(package_file)
+
+        mapper = TabularDataHelper(package)
+        attribute_map = mapper.parse_attribute_map(saved_dataset.name)
+        set_attributes(saved_dataset, attribute_map, _overwrite)
+
         saved_dataset.refresh_from_db()
         return saved_dataset
 

importer_datapackage/handlers/datapackage/handler.py

@@ -3,12 +3,24 @@
 
 import xml.etree.cElementTree as ET
 
-class SchemaToVrtMapper():
+class TabularDataHelper():
 
     def __init__(self, package):
         self.package = package
         self.resources = package.resources or []
 
+    
+    def parse_attribute_map(self, resource_name: str) -> list:
+        """ Of type: [ [field, ftype, description, label, display_oder], ... ]"""
+        
+        resource = self.package.get_resource(resource_name)
+        schema = resource.schema
+        attribute_map = [
+            [field.name, _parse_field_type(field)[0], field.description, field.title or field.name, None]
+            for field in schema.fields
+        ]
+        return attribute_map
+        
     def write_vrt_file(self, filename: str, folder: Path):
 
         if not filename:
@@ -23,7 +35,7 @@ def write_vrt_file(self, filename: str, folder: Path):
             for field in schema.fields:
                 (type, subtype) = _parse_field_type(field)
                 normalized_fieldname = normalize(field.name).lower()
-                ET.SubElement(layer, "Field", src=field.name, name=normalized_fieldname, type=type, subtype=subtype)
+                ET.SubElement(layer, "Field", src=field.name, name=normalized_fieldname, type=type, subtype=subtype,)
 
         # write VRT file
         vrt_filename = Path(folder, filename) if folder else filename

importer_datapackage/handlers/datapackage/mapper.py

@@ -12,7 +12,7 @@
 from geonode.storage.manager import StorageManager
 
 from importer_datapackage.handlers.datapackage.handler import DataPackageFileHandler
-from importer_datapackage.handlers.datapackage.mapper import SchemaToVrtMapper
+from importer_datapackage.handlers.datapackage.mapper import TabularDataHelper
 from importer_datapackage.handlers.datapackage.exceptions import InvalidDataPackageFileException
 
 def _absolute_path(filename: str):
@@ -37,7 +37,7 @@ class TestSchemaToVrtMapper(TestCase):
     def setUpClass(cls) -> None:
         super().setUpClass()
         cls.package = Package(_absolute_path("data/datapackage.json"))
-        cls.mapper = SchemaToVrtMapper(cls.package)
+        cls.mapper = TabularDataHelper(cls.package)
 
     def test_write_vrt_file(self):
         with TemporaryDirectory() as tmp_dir: