feat(projects): save results in DynamoDB - EUBFR-32 (#14)

* feat(projects): save results in DynamoDB - EUBFR-32 * Save to DB * Remove records when the file is deleted * Remove junk * Batch Delete Items * Remove dead code * Add HTTP endpoint to retrieve the list of all projects
ec-europa · Oct 5, 2017 · 13a3808 · 13a3808
1 parent 32b1310
commit 13a3808
Show file tree

Hide file tree

Showing 27 changed files with 471 additions and 231 deletions.
diff --git a/lib/dynamodb/actions/save.js b/lib/dynamodb/actions/save.js
diff --git a/lib/dynamodb/helpers/normalize.js b/lib/dynamodb/helpers/normalize.js
diff --git a/lib/dynamodb/index.js b/lib/dynamodb/index.js
diff --git a/lib/dynamodb/package.json b/lib/dynamodb/package.json
diff --git a/lib/dynamodb/tables/projects.js b/lib/dynamodb/tables/projects.js
diff --git a/package.json b/package.json
@@ -24,7 +24,7 @@
     "lerna": "2.3.1",
     "lint-staged": "4.2.3",
     "npm-run-all": "4.1.1",
-    "prettier": "1.7.3"
+    "prettier": "1.7.4"
   },
-  "workspaces": ["lib/*", "services/**"]
+  "workspaces": ["services/**"]
 }
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
@@ -34,3 +34,7 @@ cd ../cleaner
 # Deploy ETL
 cd ../etl/budg/csv
 ./node_modules/.bin/serverless deploy -v
+
+# Deploy value store - projects
+cd ../../../../value-store/projects
+./node_modules/.bin/serverless deploy -v
diff --git a/services/ingestion/cleaner/package.json b/services/ingestion/cleaner/package.json
@@ -7,15 +7,15 @@
     "test:unit": "jest --testPathPattern=unit"
   },
   "devDependencies": {
-    "aws-sdk": "2.125.0",
+    "aws-sdk": "2.127.0",
     "babel-core": "6.26.0",
     "babel-loader": "7.1.2",
     "babel-plugin-transform-runtime": "6.23.0",
     "babel-preset-es2015": "6.24.1",
     "babel-preset-stage-3": "6.24.1",
     "jest": "21.2.1",
     "serverless": "1.23.0",
-    "serverless-webpack": "3.1.0",
+    "serverless-webpack": "3.1.1",
     "webpack": "3.6.0"
   },
   "dependencies": {

diff --git a/services/ingestion/cleaner/src/handler.js b/services/ingestion/cleaner/src/handler.js
@@ -27,7 +27,7 @@ export const onObjectRemoved = (event, context, callback) => {
 
   const params = {
     Bucket: BUCKET,
-    Key: `${s3record.s3.object.key}.data`,
+    Key: `${s3record.s3.object.key}.ndjson`,
   };
   console.log('params', params);
 

diff --git a/services/ingestion/etl/budg/csv/package.json b/services/ingestion/etl/budg/csv/package.json
@@ -7,15 +7,15 @@
     "test:unit": "jest --testPathPattern=unit"
   },
   "devDependencies": {
-    "aws-sdk": "2.125.0",
+    "aws-sdk": "2.127.0",
     "babel-core": "6.26.0",
     "babel-loader": "7.1.2",
     "babel-plugin-transform-runtime": "6.23.0",
     "babel-preset-es2015": "6.24.1",
     "babel-preset-stage-3": "6.24.1",
     "jest": "21.2.1",
     "serverless": "1.23.0",
-    "serverless-webpack": "3.1.0",
+    "serverless-webpack": "3.1.1",
     "webpack": "3.6.0"
   },
   "dependencies": {

diff --git a/services/ingestion/etl/budg/csv/serverless.yml b/services/ingestion/etl/budg/csv/serverless.yml
@@ -36,6 +36,7 @@ functions:
   parseCsv:
     handler: src/handler.parseCsv
     name: ${self:provider.stage}-${self:service}-parseCsv
+    memorySize: 256
     environment:
       BUCKET: ${self:custom.bucketName}
     events:

diff --git a/services/ingestion/etl/budg/csv/src/handler.js b/services/ingestion/etl/budg/csv/src/handler.js
@@ -48,7 +48,7 @@ export const parseCsv = (event, context, callback) => {
   const transformer = transform(
     (record, cb) => {
       const data = transformRecord(record);
-      cb(null, `${JSON.stringify(data)}\r\n`);
+      cb(null, `${JSON.stringify(data)}\n`);
     },
     { parallel: 10 }
   );
@@ -59,8 +59,9 @@ export const parseCsv = (event, context, callback) => {
 
     const params = {
       Bucket: BUCKET,
-      Key: `${message.object.key}.data`,
+      Key: `${message.object.key}.ndjson`,
       Body: pass,
+      ContentType: 'application/x-ndjson',
     };
 
     s3.upload(params, err => {

diff --git a/services/ingestion/etl/budg/csv/src/transform.js b/services/ingestion/etl/budg/csv/src/transform.js
@@ -53,6 +53,7 @@ export default record => {
 
   // Map the fields
   return {
+    project_id: record.Nid,
     title: record.Name,
     cover_image: record.Visual,
     programme_name: record['Programme name'],

diff --git a/services/ingestion/manager/package.json b/services/ingestion/manager/package.json
@@ -7,15 +7,15 @@
     "test:unit": "jest --testPathPattern=unit"
   },
   "devDependencies": {
-    "aws-sdk": "2.125.0",
+    "aws-sdk": "2.127.0",
     "babel-core": "6.26.0",
     "babel-loader": "7.1.2",
     "babel-plugin-transform-runtime": "6.23.0",
     "babel-preset-es2015": "6.24.1",
     "babel-preset-stage-3": "6.24.1",
     "jest": "21.2.1",
     "serverless": "1.23.0",
-    "serverless-webpack": "3.1.0",
+    "serverless-webpack": "3.1.1",
     "webpack": "3.6.0"
   },
   "dependencies": {

diff --git a/services/storage/meta-index/package.json b/services/storage/meta-index/package.json
@@ -7,7 +7,7 @@
     "test:unit": "jest --testPathPattern=unit"
   },
   "devDependencies": {
-    "aws-sdk": "2.125.0",
+    "aws-sdk": "2.127.0",
     "babel-core": "6.26.0",
     "babel-loader": "7.1.2",
     "babel-plugin-transform-runtime": "6.23.0",
@@ -16,7 +16,7 @@
     "jest": "21.2.1",
     "serverless": "1.23.0",
     "serverless-dynamodb-autoscaling": "0.6.2",
-    "serverless-webpack": "3.1.0",
+    "serverless-webpack": "3.1.1",
     "webpack": "3.6.0"
   },
   "dependencies": {

diff --git a/services/storage/meta-index/src/handler.js b/services/storage/meta-index/src/handler.js
@@ -38,6 +38,7 @@ export const onObjectCreated = (event, context, callback) => {
       // Save record
       const documentClient = new AWS.DynamoDB.DocumentClient({
         apiVersion: '2012-08-10',
+        convertEmptyValues: true,
       });
 
       const params = {

diff --git a/services/storage/signed-uploads/package.json b/services/storage/signed-uploads/package.json
@@ -8,15 +8,15 @@
     "start": "sls offline start"
   },
   "devDependencies": {
-    "aws-sdk": "2.125.0",
+    "aws-sdk": "2.127.0",
     "babel-core": "6.26.0",
     "babel-loader": "7.1.2",
     "babel-plugin-transform-runtime": "6.23.0",
     "babel-preset-es2015": "6.24.1",
     "babel-preset-stage-3": "6.24.1",
     "serverless": "1.23.0",
     "serverless-offline": "3.16.0",
-    "serverless-webpack": "3.1.0",
+    "serverless-webpack": "3.1.1",
     "webpack": "3.6.0"
   },
   "dependencies": {

diff --git a/services/storage/signed-uploads/src/upsert.js b/services/storage/signed-uploads/src/upsert.js
@@ -1,6 +1,6 @@
 /* eslint-disable import/prefer-default-export */
 
-import AWS from 'aws-sdk';
+import AWS from 'aws-sdk'; // eslint-disable-line import/no-extraneous-dependencies
 
 const bucket = process.env.BUCKET;
 const region = process.env.REGION;

diff --git a/services/value-store/projects/.babelrc b/services/value-store/projects/.babelrc
@@ -0,0 +1,4 @@
+{
+  "plugins": ["transform-runtime"],
+  "presets": ["es2015", "stage-3"]
+}
diff --git a/services/value-store/projects/README.md b/services/value-store/projects/README.md
@@ -0,0 +1 @@
+# Storage - Meta Index