feat(elasticsearch): improve mappings - EUBFR-88 (#55)

* feat(elasticsearch): add mappings * feat(elasticsearch): add mappings * Remove google shortener
ec-europa · Dec 13, 2017 · 8a67c11 · 8a67c11
1 parent 34fdb1e
commit 8a67c11
Show file tree

Hide file tree

Showing 5 changed files with 59 additions and 31 deletions.
diff --git a/services/ingestion/etl/agri/csv/src/lib/transform.js b/services/ingestion/etl/agri/csv/src/lib/transform.js
@@ -61,7 +61,8 @@ export default (record: Object): Project => {
       country_name: null,
       country_code: country,
       location: {
-        // elasticsearch specific structure for geo_point https://goo.gl/nbi2Yp
+        // elasticsearch specific structure for geo_point
+        // read more https://www.elastic.co/guide/en/elasticsearch/reference/5.5/modules-scripting-expression.html#_literal_geo_point_literal_field_api
         lat: (Array.isArray(latArray) && latArray[index]) || 0,
         lon: (Array.isArray(longArray) && longArray[index]) || 0,
       },

diff --git a/services/value-store/elasticsearch/serverless.yml b/services/value-store/elasticsearch/serverless.yml
@@ -53,7 +53,7 @@ resources:
               Service: lambda.amazonaws.com
             Action: sts:AssumeRole
           # Admin access to Kibana from an IP
-          # https://goo.gl/eiGgpD
+          # http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-kibana.html?shortFooter=true
           - Effect: Allow
             Principal: "*"
             Action: es:*

diff --git a/services/value-store/projects/serverless.yml b/services/value-store/projects/serverless.yml
@@ -25,7 +25,7 @@ provider:
   stackTags:
     ENV: ${self:custom.eubfrEnvironment}
   iamRoleStatements:
-    # https://goo.gl/U21zxP
+    # http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-aws-integrations.html?shortFooter=true#es-aws-integrations-s3-lambda-es-authorizations
     - Effect: "Allow"
       Action: "es:*"
       Resource: "arn:aws:es:${self:provider.region}:*:domain/*"

diff --git a/services/value-store/projects/src/events/onObjectCreated.js b/services/value-store/projects/src/events/onObjectCreated.js
@@ -7,6 +7,8 @@ import split2 from 'split2';
 
 import deleteProjects from '../lib/deleteProjects';
 import SaveStream from '../lib/SaveStream';
+// elasticsearch mapping https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
+import ProjectMapping from '../mappings/project';
 
 export const handler = (event, context, callback) => {
   const { API, INDEX } = process.env;
@@ -45,33 +47,6 @@ export const handler = (event, context, callback) => {
     index: INDEX,
   };
 
-  // elasticsearch mapping overrides/corrections
-  // the rest of the fields are dynamically discovered fine for now
-  const body = {
-    project: {
-      properties: {
-        project_locations: {
-          properties: {
-            location: {
-              type: 'geo_point',
-              ignore_malformed: true,
-            },
-          },
-        },
-        timeframe: {
-          properties: {
-            from: {
-              type: 'date',
-            },
-            to: {
-              type: 'date',
-            },
-          },
-        },
-      },
-    },
-  };
-
   // elasticsearch client instantiation
   const client = elasticsearch.Client(options);
 
@@ -84,7 +59,9 @@ export const handler = (event, context, callback) => {
       return exists;
     })
     .then(() => client.indices.getMapping({ index: INDEX, type }))
-    .catch(() => client.indices.putMapping({ index: INDEX, type, body }))
+    .catch(() =>
+      client.indices.putMapping({ index: INDEX, type, body: ProjectMapping })
+    )
     .then(() =>
       s3
         .headObject({

diff --git a/services/value-store/projects/src/mappings/project.js b/services/value-store/projects/src/mappings/project.js
@@ -0,0 +1,50 @@
+module.exports = {
+  project: {
+    properties: {
+      budget: {
+        properties: {
+          funding_area: {
+            type: 'keyword',
+          },
+        },
+      },
+      computed_key: {
+        type: 'keyword',
+      },
+      coordinators: {
+        properties: {
+          country: {
+            type: 'keyword',
+          },
+        },
+      },
+      ec_priorities: {
+        type: 'keyword',
+      },
+      producer_id: {
+        type: 'keyword',
+      },
+      programme_name: {
+        type: 'keyword',
+      },
+      project_locations: {
+        properties: {
+          location: {
+            type: 'geo_point',
+            ignore_malformed: true,
+          },
+        },
+      },
+      timeframe: {
+        properties: {
+          from: {
+            type: 'date',
+          },
+          to: {
+            type: 'date',
+          },
+        },
+      },
+    },
+  },
+};