feat(ingestion): react to S3 events - EUBFR-20 (#5)

* feat(ingestion): react to S3 events - no issue

* Separate storage and ingestion

* Upgrade node-docker version (including Yarn v1.0.1)

* Update docs and config

* Publish message to SNS topic

* Deploy first ETL

* Make the region dynamic too

* Make ETL great again

* Add DB service

* Avoid using the same port everywhere

* Load the file and puts its content into the sns message

* Parse CSV

* One line -> one SMS message

* Update dependencies

* Update dependencies and remove serverless-offline

* Don't embed aws-sdk (already provided)

* Manually exclude externals

.drone.yml

@@ -14,25 +14,24 @@ clone:
 
 pipeline:
   install:
-    image: yhuard/node:latest
-    pull: true
+    image: node:8.5
     commands:
       - yarn install --frozen-lockfile
 
   test-lint:
-    image: yhuard/node:latest
+    image: node:8.5
     group: test
     commands:
       - yarn test:lint
 
   test-integration:
-    image: yhuard/node:latest
+    image: node:8.5
     group: test
     commands:
       - yarn test:integration
 
   test-unit:
-    image: yhuard/node:latest
+    image: node:8.5
     group: test
     commands:
       - yarn test:unit

.gitignore

@@ -2,3 +2,4 @@
 .webpack
 node_modules
 *.log
+config.json

.nvmrc

@@ -1 +1 @@
-8
+8.5

README.md

@@ -24,8 +24,39 @@ Setup your environment:
 yarn
 ```
 
-Test the services offline:
+## Configuration
+
+Copy `config.example.json` to `config.json` and set the values according to your environment.
+
+## Test
+
+Test the services:
 
 ```sh
-yarn offline
+yarn test
 ```
+
+## Deploy
+
+First, you need to [set up your AWS credentials](https://serverless.com/framework/docs/providers/aws/guide/credentials/).
+
+### All at once
+
+Go to the project's root and type:
+
+```sh
+yarn deploy
+```
+
+### Manually
+
+Go to the folder of the service you want to deploy and type:
+
+```sh
+yarn deploy
+```
+
+## Notes
+
+-   Deploy `services/storage` first.
+-   Keep the same stage name for every services so they can work together.

config.example.json

@@ -0,0 +1,4 @@
+{
+  "region": "eu-central-1",
+  "stage": "dev-username-uid"
+}

lerna.json

@@ -1,5 +1,5 @@
 {
-  "lerna": "2.1.2",
+  "lerna": "2.2.0",
   "version": "0.0.1",
   "npmClient": "yarn",
   "useWorkspaces": true

package.json

@@ -2,9 +2,8 @@
   "private": true,
   "scripts": {
     "bootstrap": "lerna bootstrap",
-    "deploy": "lerna exec -- serverless deploy -v",
+    "deploy": "./scripts/deploy.sh",
     "lint": "eslint .",
-    "offline": "lerna run --parallel offline",
     "precommit": "lint-staged",
     "test:integration": "lerna run --parallel test:integration",
     "test:unit": "lerna run --parallel test:unit",
@@ -15,17 +14,17 @@
     "*.{js,json}": ["prettier --write", "git add"]
   },
   "devDependencies": {
-    "eslint": "4.6.1",
+    "eslint": "4.7.0",
     "eslint-config-airbnb-base": "12.0.0",
-    "eslint-config-prettier": "2.4.0",
+    "eslint-config-prettier": "2.5.0",
     "eslint-plugin-import": "2.7.0",
-    "eslint-plugin-jest": "21.0.2",
-    "eslint-plugin-prettier": "2.2.0",
+    "eslint-plugin-jest": "21.1.0",
+    "eslint-plugin-prettier": "2.3.0",
     "husky": "0.14.3",
-    "lerna": "2.1.2",
-    "lint-staged": "4.1.3",
+    "lerna": "2.2.0",
+    "lint-staged": "4.2.1",
     "npm-run-all": "4.1.1",
-    "prettier": "1.6.1"
+    "prettier": "1.7.0"
   },
-  "workspaces": ["services/*"]
+  "workspaces": ["services/**"]
 }

scripts/deploy.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# Exit the script on any command with non 0 return code
+set -e
+
+# Go to project root
+cd "$(dirname "$0")"
+cd ..
+
+# Deploy storage first
+cd ./services/storage
+./node_modules/.bin/serverless deploy -v
+
+# Go to project root
+cd ../..
+
+# Deploy manager
+cd ./services/ingestion/manager
+./node_modules/.bin/serverless deploy -v
+
+# Go to project root
+cd ../../..
+
+# Deploy ETL
+cd ./services/ingestion/etl/budg/csv
+./node_modules/.bin/serverless deploy -v
+
+# Go to project root
+cd ../../../../..
+
+# Deploy DB service
+cd ./services/db
+./node_modules/.bin/serverless deploy -v

services/db/README.md

@@ -0,0 +1 @@
+# Ingestion DB

services/db/package.json

@@ -0,0 +1,24 @@
+{
+  "private": true,
+  "name": "db",
+  "version": "0.0.1",
+  "scripts": {
+    "deploy": "sls deploy -v",
+    "test:unit": "jest --testPathPattern=unit"
+  },
+  "devDependencies": {
+    "babel-core": "6.26.0",
+    "babel-loader": "7.1.2",
+    "babel-plugin-transform-runtime": "6.23.0",
+    "babel-preset-es2015": "6.24.1",
+    "babel-preset-stage-3": "6.24.1",
+    "jest": "21.1.0",
+    "serverless": "1.22.0",
+    "serverless-webpack": "3.0.0",
+    "webpack": "3.6.0"
+  },
+  "dependencies": {
+    "aws-sdk": "2.117.0",
+    "babel-runtime": "6.26.0"
+  }
+}

services/db/serverless.yml

@@ -0,0 +1,60 @@
+service: db
+
+plugins:
+  - serverless-webpack
+
+custom:
+  webpackIncludeModules: true
+  eubfrEnvironment: ${env:EUBFR_ENV, 'dev'}
+
+package:
+  individually: true
+
+provider:
+  name: aws
+  runtime: nodejs6.10
+  stage: ${opt:stage, file(../../config.json):stage, 'dev'}
+  region: ${opt:region, file(../../config.json):region, 'eu-central-1'}
+  deploymentBucket:
+    name: eubfr-${self:custom.eubfrEnvironment}-deploy
+  stackTags:
+    ENV: ${self:custom.eubfrEnvironment}
+
+functions:
+  save:
+    handler: src/handler.save
+    events:
+      - sns:
+          arn:
+            Fn::Join:
+              - ""
+              - - "arn:aws:sns:"
+                - Ref: "AWS::Region"
+                - ":"
+                - Ref: "AWS::AccountId"
+                - ":${self:provider.stage}-db"
+          topicName: ${self:provider.stage}-db
+
+resources:
+  Resources:
+    SNSTopicDB:
+      Type: AWS::SNS::Topic
+      Properties:
+        TopicName: ${self:provider.stage}-db
+        DisplayName: Save to DB
+    SNSTopicDBPolicy:
+      Type: AWS::SNS::TopicPolicy
+      Properties:
+        PolicyDocument:
+          Version: '2012-10-17'
+          Statement:
+            -
+              Sid: AllowAllToPushNotificationEffect
+              Action:
+                - sns:Publish
+              Effect: Allow
+              Resource: "*"
+              Principal: "*"
+        Topics:
+          -
+            Ref: SNSTopicDB

services/db/src/handler.js

@@ -0,0 +1,18 @@
+/* eslint-disable import/prefer-default-export, no-console */
+export const save = (event, context, callback) => {
+  /*
+   * Some checks here before going any further
+   */
+
+  // Only work on the first record
+  const snsRecord = event.Records[0];
+
+  // Was the lambda triggered correctly? Is the file extension supported? etc.
+  if (!snsRecord || snsRecord.EventSource !== 'aws:sns') {
+    return callback('Bad record');
+  }
+
+  // Save to DB
+
+  return callback(null, 'All fine');
+};

services/example01/webpack.config.js → services/db/webpack.config.js

@@ -1,12 +1,11 @@
 /* eslint-disable import/no-extraneous-dependencies */
 const slsw = require('serverless-webpack');
-const nodeExternals = require('webpack-node-externals');
 const path = require('path');
 
 module.exports = {
   entry: slsw.lib.entries,
   target: 'node',
-  externals: [nodeExternals()],
+  externals: [{ 'aws-sdk': true }],
   module: {
     loaders: [
       {