feat(meta-index): replace dynamodb with elasticsearch - EUBFR-104 (#72)

* feat(meta-index): replace dynamodb with elasticsearch - EUBFR-104 * Add meta index * Fix onObjectCreated * Update demo * Fetch only current producer's files * Meta delete * Update status + sort by date
ec-europa · Jan 18, 2018 · 7f531e7 · 7f531e7
1 parent 60d2b41
commit 7f531e7
Show file tree

Hide file tree

Showing 25 changed files with 304 additions and 432 deletions.
diff --git a/demo/dashboard/client/serverless.yml b/demo/dashboard/client/serverless.yml
@@ -22,12 +22,15 @@ provider:
     ENV: ${self:custom.eubfrEnvironment}
   environment:
     REACT_APP_STAGE: ${self:provider.stage}
+    REACT_APP_PRODUCER: ${self:custom.EUBFR_USERNAME}
     REACT_APP_DEMO_SERVER:
       Fn::ImportValue: ${self:provider.stage}:demo-dash-server-${self:custom.EUBFR_USERNAME}:ServiceEndpoint
     REACT_APP_ES_PROJECTS_ENDPOINT:
       Fn::ImportValue: ${self:custom.eubfrEnvironment}:resources-elasticsearch:ProjectsEndpoint
     REACT_APP_ES_LOGS_ENDPOINT:
       Fn::ImportValue: ${self:custom.eubfrEnvironment}:resources-elasticsearch:LogsEndpoint
+    REACT_APP_ES_META_ENDPOINT:
+      Fn::ImportValue: ${self:custom.eubfrEnvironment}:resources-elasticsearch:MetaEndpoint
 
 resources:
   Resources:

diff --git a/demo/dashboard/client/src/App.js b/demo/dashboard/client/src/App.js
@@ -33,7 +33,9 @@ const App = () => (
             </ol>
           </nav>
           <div className="ecl-page-header__body">
-            <div className="ecl-page-header__identity">Dashboard</div>
+            <div className="ecl-page-header__identity">
+              {(process.env.REACT_APP_PRODUCER || '').toUpperCase()} Dashboard
+            </div>
           </div>
         </div>
       </div>

diff --git a/demo/dashboard/client/src/components/FilesList.js b/demo/dashboard/client/src/components/FilesList.js
@@ -21,24 +21,27 @@ const List = ({ files }) => (
       </tr>
     </thead>
     <tbody>
-      {files.map(file => (
-        <tr key={file.computed_key}>
-          <td>
-            <span
-              title={file.message || 'Not parsed'}
-              className={getIcon(file.status)}
-            />
-            <Link
-              to={`/files/${encodeURIComponent(file.computed_key)}`}
-              className="ecl-link"
-            >
-              {file.original_key || 'unknown'}
-            </Link>
-          </td>
-          <td>{new Date(file.last_modified).toLocaleString()}</td>
-          <td>{Math.floor(file.content_length / 1024) || 0} kB</td>
-        </tr>
-      ))}
+      {files.map(file => {
+        const hit = file._source;
+        return (
+          <tr key={hit.computed_key}>
+            <td>
+              <span
+                title={hit.message || 'Not parsed'}
+                className={getIcon(hit.status)}
+              />
+              <Link
+                to={`/files/${encodeURIComponent(hit.computed_key)}`}
+                className="ecl-link"
+              >
+                {hit.original_key || 'unknown'}
+              </Link>
+            </td>
+            <td>{new Date(hit.last_modified).toLocaleString()}</td>
+            <td>{Math.floor(hit.content_length / 1024) || 0} kB</td>
+          </tr>
+        );
+      })}
     </tbody>
   </table>
 );

diff --git a/demo/dashboard/client/src/pages/files/File.js b/demo/dashboard/client/src/pages/files/File.js
@@ -11,6 +11,10 @@ import ProjectsTab from './file/Projects';
 import './File.css';
 
 const demoServerEndpoint = `https://${process.env.REACT_APP_DEMO_SERVER}/demo`;
+
+const metaApiEndpoint = `https://${process.env.REACT_APP_ES_META_ENDPOINT}`;
+const metaIndex = `${process.env.REACT_APP_STAGE}-meta`;
+
 const projectsApiEndpoint = `https://${
   process.env.REACT_APP_ES_PROJECTS_ENDPOINT
 }`;
@@ -38,15 +42,22 @@ class File extends React.Component {
       projectsCountLoading: true,
     };
 
+    this.projectsClient = null;
+    this.metaClient = null;
     this.deleteFile = this.deleteFile.bind(this);
     this.generateLink = this.generateLink.bind(this);
-    this.getFileMeta = this.getFileMeta.bind(this);
     this.loadFile = this.loadFile.bind(this);
     this.getProjectsCount = this.getProjectsCount.bind(this);
   }
 
   componentDidMount() {
-    this.client = elasticsearch.Client({
+    this.metaClient = elasticsearch.Client({
+      host: metaApiEndpoint,
+      apiVersion: '6.0',
+      log: 'warning',
+    });
+
+    this.projectsClient = elasticsearch.Client({
       host: projectsApiEndpoint,
       apiVersion: '6.0',
       log: 'warning',
@@ -60,7 +71,7 @@ class File extends React.Component {
     const { match } = this.props;
     const computedKey = decodeURIComponent(match.params.id);
 
-    return this.client
+    return this.projectsClient
       .count({
         index: projectsIndex,
         type: 'project',
@@ -77,36 +88,27 @@ class File extends React.Component {
       });
   }
 
-  getFileMeta(computedKey) {
-    return () =>
-      window
-        .fetch(
-          `${demoServerEndpoint}/filemeta?key=${encodeURIComponent(
-            computedKey
-          )}`
-        )
-        .then(handleErrors)
-        .then(response => response.json())
+  loadFile() {
+    const { match } = this.props;
+    const computedKey = decodeURIComponent(match.params.id);
+
+    this.setState({ fileLoading: true }, () =>
+      this.metaClient
+        .search({
+          index: metaIndex,
+          type: 'file',
+          q: `computed_key:"${computedKey}"`,
+        })
         .then(data =>
           this.setState({
             fileLoading: false,
-            file: data[0],
+            file: data.hits.hits[0]._source,
           })
         )
         .catch(error => {
-          console.log(`An error occured: ${error.message}`);
-        });
-  }
-
-  loadFile() {
-    this.setState({
-      fileLoading: true,
-    });
-
-    const { match } = this.props;
-    const computedKey = decodeURIComponent(match.params.id);
-
-    this.setState({ fileLoading: true }, this.getFileMeta(computedKey));
+          throw Error(`An error occured: ${error.message}`);
+        })
+    );
   }
 
   deleteFile() {

diff --git a/demo/dashboard/client/src/pages/files/List.js b/demo/dashboard/client/src/pages/files/List.js
@@ -1,10 +1,11 @@
 import React, { Component } from 'react';
+import elasticsearch from 'elasticsearch';
 import FilesList from '../../components/FilesList';
-import handleErrors from '../../lib/handleErrors';
 
 import Spinner from '../../components/Spinner';
 
-const demoServer = `https://${process.env.REACT_APP_DEMO_SERVER}/demo`;
+const metaApiEndpoint = `https://${process.env.REACT_APP_ES_META_ENDPOINT}`;
+const metaIndex = `${process.env.REACT_APP_STAGE}-meta`;
 
 class List extends Component {
   constructor() {
@@ -13,33 +14,51 @@ class List extends Component {
     this.state = {
       loading: true,
       files: [],
+      filesCount: 0,
     };
 
+    this.esClient = null;
     this.loadFiles = this.loadFiles.bind(this);
   }
 
   componentDidMount() {
+    this.esClient = elasticsearch.Client({
+      host: metaApiEndpoint,
+      apiVersion: '6.0',
+      log: 'warning',
+    });
+
     this.loadFiles();
   }
 
   loadFiles() {
-    this.setState(
+    return this.setState(
       {
         loading: true,
       },
       () =>
-        window
-          .fetch(`${demoServer}/meta`)
-          .then(handleErrors)
-          .then(response => response.json())
+        this.esClient
+          .search({
+            index: metaIndex,
+            type: 'file',
+            body: {
+              query: {
+                term: {
+                  producer_id: process.env.REACT_APP_PRODUCER,
+                },
+              },
+              sort: [{ last_modified: { order: 'desc' } }],
+            },
+          })
           .then(data =>
             this.setState({
               loading: false,
-              files: data,
+              files: data.hits.hits,
+              filesCount: data.hits.total,
             })
           )
           .catch(error => {
-            console.log(`An error happened: ${error.message}`);
+            throw Error(`An error occured: ${error.message}`);
           })
     );
   }

diff --git a/demo/dashboard/client/src/pages/files/file/Logs.js b/demo/dashboard/client/src/pages/files/file/Logs.js
@@ -18,6 +18,7 @@ class Logs extends React.Component {
       logsCount: 0,
     };
 
+    this.logClient = null;
     this.loadLogs = this.loadLogs.bind(this);
     this.setLogs = this.setLogs.bind(this);
     this.setEmptyLogs = this.setEmptyLogs.bind(this);

diff --git a/resources/elasticsearch/mappings/meta.js b/resources/elasticsearch/mappings/meta.js
@@ -0,0 +1,17 @@
+// Specific syntax:
+// https://serverless.com/framework/docs/providers/aws/guide/variables/#reference-variables-in-javascript-files
+
+module.exports = () => ({
+  file: {
+    properties: {
+      producer_id: { type: 'keyword' },
+      computed_key: { type: 'keyword' },
+      original_key: { type: 'keyword' },
+      producer_arn: { type: 'keyword' },
+      content_type: { type: 'keyword' },
+      last_modified: { type: 'date' },
+      content_length: { type: 'long' },
+      // metadata: otherMeta,
+    },
+  },
+});
diff --git a/resources/elasticsearch/serverless.yml b/resources/elasticsearch/serverless.yml
@@ -7,6 +7,7 @@ custom:
   eubfrEnvironment: ${opt:eubfr_env, file(../../config.json):eubfr_env, env:EUBFR_ENV, 'dev'}
   projectsDomain: ${self:custom.eubfrEnvironment}-projects
   logsDomain: ${self:custom.eubfrEnvironment}-logs
+  metaDomain: ${self:custom.eubfrEnvironment}-meta
   slsEsIndices:
     -
       type: project
@@ -20,6 +21,12 @@ custom:
       mapping: ${file(./mappings/logs.js)}
       region: ${opt:region, file(../../config.json):region, 'eu-central-1'}
       endpointName: ${self:custom.eubfrEnvironment}:resources-elasticsearch:LogsEndpoint
+    -
+      type: file
+      index: ${opt:stage, file(../../config.json):stage, 'dev'}-meta
+      mapping: ${file(./mappings/meta.js)}
+      region: ${opt:region, file(../../config.json):region, 'eu-central-1'}
+      endpointName: ${self:custom.eubfrEnvironment}:resources-elasticsearch:MetaEndpoint
 
 
 provider:
@@ -118,6 +125,48 @@ resources:
                 aws:SourceIp:
                 - 147.67.241.226
             Resource: "arn:aws:es:${self:provider.region}:*:domain/${self:custom.logsDomain}/*"
+    MetaElasticSearchDomain:
+      Type: AWS::Elasticsearch::Domain
+      # Don't delete the elastic search instance when the CF stack is deleted
+      DeletionPolicy: Retain
+      Properties:
+        DomainName: ${self:custom.metaDomain}
+        ElasticsearchVersion: '6.0'
+        EBSOptions:
+          EBSEnabled: true
+          VolumeType: gp2
+          VolumeSize: 10
+        ElasticsearchClusterConfig:
+          InstanceType: t2.small.elasticsearch
+          InstanceCount: 1
+          DedicatedMasterEnabled: false
+          ZoneAwarenessEnabled: false
+        AccessPolicies:
+          Version: '2012-10-17'
+          Statement:
+          # Public can query for information
+          - Effect: Allow
+            Principal: "*"
+            Action:
+             - "es:ESHttpHead"
+             - "es:ESHttpGet"
+             - "es:ESHttpPost"
+            Resource: "arn:aws:es:${self:provider.region}:*:domain/${self:custom.metaDomain}/*"
+          # Lambda can take actions on the ES domain
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: sts:AssumeRole
+          # Admin access to Kibana from an IP
+          # https://goo.gl/eiGgpD
+          - Effect: Allow
+            Principal: "*"
+            Action: es:*
+            Condition:
+              IpAddress:
+                aws:SourceIp:
+                - 147.67.241.226
+            Resource: "arn:aws:es:${self:provider.region}:*:domain/${self:custom.metaDomain}/*"
   Outputs:
     ProjectsEndpoint:
       Description: The API endpoint of the projects' elasticsearch domain.
@@ -133,3 +182,10 @@ resources:
       Export:
         # Global varibale, uses eubfrEnvironment instead of stage
         Name: "${self:custom.eubfrEnvironment}:${self:service}:LogsEndpoint"
+    MetaEndpoint:
+      Description: The API endpoint of the meta's elasticsearch domain.
+      Value:
+        Fn::GetAtt: ["MetaElasticSearchDomain", "DomainEndpoint"]
+      Export:
+        # Global varibale, uses eubfrEnvironment instead of stage
+        Name: "${self:custom.eubfrEnvironment}:${self:service}:MetaEndpoint"
diff --git a/services/ingestion/etl/agri/csv/src/events/onParseCSV.js b/services/ingestion/etl/agri/csv/src/events/onParseCSV.js
@@ -5,7 +5,7 @@ import transform from 'stream-transform';
 import Logger from '../../../../../../logger/listener/src/lib/Logger';
 
 // Import constants
-import { STATUS } from '../../../../../../storage/meta-index/src/events/onStatusReported';
+import { STATUS } from '../../../../../../storage/meta-index/src/lib/status';
 
 // Import logic
 import { extractMessage, prepareMessage } from '../lib/sns';

diff --git a/services/ingestion/etl/agri/csv/src/lib/uploadFromStream.js b/services/ingestion/etl/agri/csv/src/lib/uploadFromStream.js
@@ -1,5 +1,5 @@
 import stream from 'stream';
-import { STATUS } from '../../../../../../storage/meta-index/src/events/onStatusReported';
+import { STATUS } from '../../../../../../storage/meta-index/src/lib/status';
 
 // Load
 export default ({ key, BUCKET, s3, sns, endpointArn, onError, callback }) => {

diff --git a/services/ingestion/etl/budg/xls/src/events/onParseXLS.js b/services/ingestion/etl/budg/xls/src/events/onParseXLS.js
@@ -6,7 +6,7 @@ import transformRecord from '../lib/transform';
 import Logger from '../../../../../../logger/listener/src/lib/Logger';
 
 // Import constants
-import { STATUS } from '../../../../../../storage/meta-index/src/events/onStatusReported';
+import { STATUS } from '../../../../../../storage/meta-index/src/lib/status';
 
 export const handler = async (event, context, callback) => {
   /*

diff --git a/services/ingestion/etl/inforegio/json/src/events/onParseJSON.js b/services/ingestion/etl/inforegio/json/src/events/onParseJSON.js
@@ -3,7 +3,7 @@ import AWS from 'aws-sdk'; // eslint-disable-line import/no-extraneous-dependenc
 
 import Logger from '../../../../../../logger/listener/src/lib/Logger';
 import transformRecord from '../lib/transform';
-import { STATUS } from '../../../../../../storage/meta-index/src/events/onStatusReported';
+import { STATUS } from '../../../../../../storage/meta-index/src/lib/status';
 
 // Destination bucket
 const { BUCKET } = process.env;

diff --git a/services/ingestion/etl/inforegio/xml/src/events/onParseXML.js b/services/ingestion/etl/inforegio/xml/src/events/onParseXML.js
@@ -3,7 +3,7 @@ import AWS from 'aws-sdk'; // eslint-disable-line import/no-extraneous-dependenc
 
 import Logger from '../../../../../../logger/listener/src/lib/Logger';
 import transformRecord from '../lib/transform';
-import { STATUS } from '../../../../../../storage/meta-index/src/events/onStatusReported';
+import { STATUS } from '../../../../../../storage/meta-index/src/lib/status';
 
 const xml2js = require('xml2js');