feat(geo-point): add centroid to records - EUBFR-130 (#82)

* feat(geo-point): add centroid to records - EUBFR-130 * Update snapshots * `lon`, not `long` * `lon`, not `long` (pt. 2) * Remove extra `filter`
ec-europa · Jan 26, 2018 · 2faf384 · 2faf384
1 parent e01bfdf
commit 2faf384
Show file tree

Hide file tree

Showing 10 changed files with 129 additions and 78 deletions.
diff --git a/resources/elasticsearch/mappings/project.js b/resources/elasticsearch/mappings/project.js
@@ -55,6 +55,9 @@ module.exports = () => ({
       },
       project_locations: {
         properties: {
+          centroid: {
+            type: 'geo_point',
+          },
           location: {
             type: 'geo_shape',
           },

diff --git a/services/ingestion/etl/agri/csv/src/lib/transform.js b/services/ingestion/etl/agri/csv/src/lib/transform.js
@@ -40,51 +40,72 @@ export default (record: Object): Project => {
   };
 
   // Preprocess coordinators
-  const coordArray = record.Coordinators.split(';').map(coordinator => ({
-    name: coordinator,
-    type: '',
-    address: '',
-    region: '',
-    country: '',
-    website: '',
-    phone: '',
-    email: '',
-  }));
+  const coordArray = record.Coordinators.split(';')
+    .filter(coordinator => coordinator)
+    .map(coordinator => ({
+      name: coordinator,
+      type: '',
+      address: '',
+      region: '',
+      country: '',
+      website: '',
+      phone: '',
+      email: '',
+    }));
 
   // Preprocess partners
-  const partnerArray = record.Partners.split(',').map(partner => ({
-    name: partner,
-    type: '',
-    address: '',
-    region: '',
-    country: '',
-    website: '',
-  }));
+  const partnerArray = record.Partners.split(',')
+    .filter(partner => partner)
+    .map(partner => ({
+      name: partner,
+      type: '',
+      address: '',
+      region: '',
+      country: '',
+      website: '',
+    }));
 
   // Preprocess locations
   const longArray = record['Project location longitude'].split(';');
   const latArray = record['Project location latitude'].split(';');
   const locationArray = record['Project country(ies)']
     .split(';')
-    .map((country, index) => ({
-      country_code: country,
-      region: '',
-      nuts2: '',
-      address: record['Project address(es)'] || '',
-      postal_code: record['Project postal code(s)'] || '',
-      town: record['Project town(s)'] || '',
-      location: {
-        type: 'Point',
-        coordinates: [
-          parseFloat(Array.isArray(longArray) && longArray[index]) || 0,
-          parseFloat(Array.isArray(latArray) && latArray[index]) || 0,
-        ],
-      },
-    }));
+    .map((country, index) => {
+      const hasCoordinates =
+        Array.isArray(longArray) &&
+        longArray[index] &&
+        Array.isArray(latArray) &&
+        latArray[index];
+
+      return {
+        country_code: country,
+        region: '',
+        nuts2: '',
+        address: record['Project address(es)'] || '',
+        postal_code: record['Project postal code(s)'] || '',
+        town: record['Project town(s)'] || '',
+        centroid: hasCoordinates
+          ? {
+              lat: parseFloat(latArray[index]) || 0,
+              lon: parseFloat(longArray[index]) || 0,
+            }
+          : null,
+        location: hasCoordinates
+          ? {
+              type: 'Point',
+              coordinates: [
+                parseFloat(longArray[index]) || 0,
+                parseFloat(latArray[index]) || 0,
+              ],
+            }
+          : null,
+      };
+    });
 
   // Preprocess related links
   const links = (record['Related links'] || '')
     .split(';')
+    .filter(link => link)
     .map(link => {
       const matches = link.match(/<a .*href="(.*)".*>(.*)<\/a>/i);
 
@@ -130,7 +151,8 @@ export default (record: Object): Project => {
     call_year: '',
     coordinators: coordArray,
     description: record['Project description'] || '',
-    ec_priorities: record['EC’s priorities'].split(';') || [],
+    ec_priorities:
+      record['EC’s priorities'].split(';').filter(priority => priority) || [],
     media: {
       cover_image: record.Visual || '',
       video: record['Link to a video'] || '',

diff --git a/services/ingestion/etl/agri/csv/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/agri/csv/test/unit/lib/__snapshots__/transform.spec.js.snap
@@ -152,6 +152,10 @@ Object {
   "project_locations": Array [
     Object {
       "address": "",
+      "centroid": Object {
+        "lat": 48.211312,
+        "lon": 16.367315,
+      },
       "country_code": "AT",
       "location": Object {
         "coordinates": Array [
@@ -167,6 +171,10 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": Object {
+        "lat": 45.803508,
+        "lon": 16.017459,
+      },
       "country_code": "HR",
       "location": Object {
         "coordinates": Array [
@@ -182,6 +190,10 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": Object {
+        "lat": 49.449922,
+        "lon": 11.10347,
+      },
       "country_code": "DE",
       "location": Object {
         "coordinates": Array [
@@ -197,6 +209,10 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": Object {
+        "lat": 48.143047,
+        "lon": 17.12426,
+      },
       "country_code": "SK",
       "location": Object {
         "coordinates": Array [
@@ -212,6 +228,10 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": Object {
+        "lat": 46.048183,
+        "lon": 14.501456,
+      },
       "country_code": "SI",
       "location": Object {
         "coordinates": Array [
@@ -257,6 +277,10 @@ Object {
 exports[`DG AGRI CSV transformer Project location information is correctly mapped 1`] = `
 Object {
   "address": "",
+  "centroid": Object {
+    "lat": 48.211312,
+    "lon": 16.367315,
+  },
   "country_code": "AT",
   "location": Object {
     "coordinates": Array [
@@ -305,41 +329,23 @@ Object {
     },
   },
   "call_year": "",
-  "coordinators": Array [
-    Object {
-      "address": "",
-      "country": "",
-      "email": "",
-      "name": "",
-      "phone": "",
-      "region": "",
-      "type": "",
-      "website": "",
-    },
-  ],
+  "coordinators": Array [],
   "description": "",
-  "ec_priorities": Array [
-    "",
-  ],
+  "ec_priorities": Array [],
   "media": Object {
     "cover_image": "",
     "video": "",
   },
-  "partners": Array [
-    Object {
-      "address": "",
-      "country": "",
-      "name": "",
-      "region": "",
-      "type": "",
-      "website": "",
-    },
-  ],
+  "partners": Array [],
   "programme_name": "",
   "project_id": "",
   "project_locations": Array [
     Object {
       "address": "",
+      "centroid": Object {
+        "lat": 42.73806663,
+        "lon": 23.40014509,
+      },
       "country_code": "",
       "location": Object {
         "coordinates": Array [
@@ -355,12 +361,7 @@ Object {
     },
   ],
   "project_website": "",
-  "related_links": Array [
-    Object {
-      "label": "",
-      "url": "",
-    },
-  ],
+  "related_links": Array [],
   "results": Object {
     "available": "",
     "result": "",

diff --git a/services/ingestion/etl/budg/xls/src/lib/transform.js b/services/ingestion/etl/budg/xls/src/lib/transform.js
@@ -70,29 +70,31 @@ export default (record: Object): Project => {
   });
 
   const partnerArray = [];
-  for (let i = 0; i < partnerKeys.length; i += 1) {
-    if (record[`Partner ${i + 1} name`]) {
+  for (let i = 1; i <= partnerKeys.length; i += 1) {
+    if (record[`Partner ${i} name`]) {
       partnerArray.push({
-        name: record[`Partner ${i + 1} name`],
-        type: record[`Partner ${i + 1} organisation type`],
-        address: record[`Partner ${i + 1} address`],
-        region: record[`Partner ${i + 1} region`],
-        country: record[`Partner ${i + 1} country`],
-        website: record[`Partner ${i + 1} website`],
+        name: record[`Partner ${i} name`],
+        type: record[`Partner ${i} organisation type`],
+        address: record[`Partner ${i} address`],
+        region: record[`Partner ${i} region`],
+        country: record[`Partner ${i} country`],
+        website: record[`Partner ${i} website`],
       });
     }
   }
 
   // Preprocess locations
   const locationArray = record['Participating countries']
     .split(',')
+    .filter(loc => loc)
     .map(country => ({
       country_code: country,
       region: '',
       nuts2: '',
       address: '',
       postal_code: '',
       town: '',
+      centroid: null,
       location: null,
     }));
 

diff --git a/services/ingestion/etl/budg/xls/test/unit/lib/__snapshots__/transform.spec.js.snap b/services/ingestion/etl/budg/xls/test/unit/lib/__snapshots__/transform.spec.js.snap
@@ -98,6 +98,7 @@ Object {
   "project_locations": Array [
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "NL",
       "location": null,
       "nuts2": "",
@@ -107,6 +108,7 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "ES",
       "location": null,
       "nuts2": "",
@@ -116,6 +118,7 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "IT",
       "location": null,
       "nuts2": "",
@@ -125,6 +128,7 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "BA",
       "location": null,
       "nuts2": "",
@@ -134,6 +138,7 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "PL",
       "location": null,
       "nuts2": "",
@@ -143,6 +148,7 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "UK",
       "location": null,
       "nuts2": "",
@@ -270,6 +276,7 @@ Object {
   "project_locations": Array [
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "NL",
       "location": null,
       "nuts2": "",
@@ -279,6 +286,7 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "ES",
       "location": null,
       "nuts2": "",
@@ -288,6 +296,7 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "IT",
       "location": null,
       "nuts2": "",
@@ -297,6 +306,7 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "BA",
       "location": null,
       "nuts2": "",
@@ -306,6 +316,7 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "PL",
       "location": null,
       "nuts2": "",
@@ -315,6 +326,7 @@ Object {
     },
     Object {
       "address": "",
+      "centroid": null,
       "country_code": "UK",
       "location": null,
       "nuts2": "",