Merge remote-tracking branch 'origin/master' into feature/woodpecker

Author: madnificent

.babelrc.js

@@ -0,0 +1,99 @@
+---
+kind: pipeline
+type: docker
+name: push-latest
+steps:
+- name: submodules
+  image: alpine/git
+  commands:
+  - git submodule update --init --recursive
+- name: build-and-push
+  image: plugins/docker
+  settings:
+    repo: ${DRONE_REPO_NAMESPACE/mu-semtech/semtech}/${DRONE_REPO_NAME%-service}
+    username:
+      from_secret: docker_username
+    password:
+      from_secret: docker_password
+trigger:
+  branch:
+  - master
+  event:
+    exclude:
+    - pull_request
+---
+kind: pipeline
+type: docker
+name: push-feature-build
+steps:
+- name: submodules
+  image: alpine/git
+  commands:
+  - git submodule update --init --recursive
+- name: push-feature-build
+  image: plugins/docker
+  settings:
+    repo: ${DRONE_REPO_NAMESPACE/mu-semtech/semtech}/${DRONE_REPO_NAME%-service}
+    tags: ${DRONE_BRANCH/\//-}
+    username:
+      from_secret: docker_username
+    password:
+      from_secret: docker_password
+    purge: true
+trigger:
+  branch:
+    - "*/*"
+  event:
+    exclude:
+      - pull_request
+---
+kind: pipeline
+type: docker
+name: push-release
+steps:
+- name: submodules
+  image: alpine/git
+  commands:
+  - git submodule update --init --recursive
+- name: build-and-push-tag
+  image: plugins/docker
+  settings:
+    repo: ${DRONE_REPO_NAMESPACE/mu-semtech/semtech}/${DRONE_REPO_NAME%-service}
+    tags: ${DRONE_TAG##v} # strips v from the tag
+    username:
+      from_secret: docker_username
+    password:
+      from_secret: docker_password
+trigger:
+  event:
+  - tag
+---
+kind: pipeline
+type: docker
+name: dry-run
+steps:
+- name: submodules
+  image: alpine/git
+  commands:
+  - git submodule update --init --recursive
+- name: dry-run
+  image: plugins/docker
+  settings:
+    repo: ${DRONE_REPO_NAMESPACE/mu-semtech/semtech}/${DRONE_REPO_NAME%-service}
+    dry_run: true
+trigger:
+  event:
+  - pull_request
+---
+kind: secret
+name: docker_username
+data: kurKh/EctlKfrAViJSn3voT4duP9aBdrip7OQV7ZuS5HYPTVxMxy
+---
+kind: secret
+name: docker_password
+data: og9cL7Gfx5E0qWJs0PeDaqIH6fr3Z/aK1YmVlmsh2B8OC2f8kyZFMiGeLDp/dcLloZ2OX7HLFy3euhFR4O4Z7w==
+---
+kind: signature
+hmac: adce33d1d58f886a57a5877695d367e5c6b3347045281b61e5a18f49cf960595
+
+...

.drone.yml

@@ -8,6 +8,9 @@ ENV MU_SPARQL_ENDPOINT 'http://database:8890/sparql'
 ENV MU_APPLICATION_GRAPH 'http://mu.semte.ch/application'
 ENV NODE_ENV 'production'
 
+ENV HOST '0.0.0.0'
+ENV PORT '80'
+
 ENV LOG_SPARQL_ALL 'true'
 ENV DEBUG_AUTH_HEADERS 'true'
 
@@ -19,6 +22,8 @@ COPY . /usr/src/app
 RUN chmod +x /usr/src/app/run-development.sh
 RUN chmod +x /usr/src/app/build-production.sh
 
+EXPOSE ${PORT}
+
 CMD sh boot.sh
 
 # This stuff only runs when building an image from the template

Dockerfile

@@ -60,6 +60,7 @@ The following importable variables are available:
   - `sparql`: [Template tag](https://www.npmjs.com/package/sparql-client-2#using-the-sparql-template-tag) to create queries with interpolated values
   - `sparqlEscapeString(value) => string`: Function to escape a string in SPARQL
   - `sparqlEscapeUri(value) => string`: Function to escape a URI in SPARQL
+  - `sparqlEscapeDecimal(value) => string`: Function to escape an integer or float as an `xsd:decimal` in SPARQL
   - `sparqlEscapeInt(value) => string`: Function to escape an integer in SPARQL
   - `sparqlEscapeFloat(value) => string`: Function to escape a float in SPARQL
   - `sparqlEscapeDate(value) => string`: Function to escape a date in SPARQL. The given value is passed to the `Date` constructor.
@@ -88,16 +89,37 @@ mu.app.get('/', function( req, res ) {
   res.send('Hello using full import');
 } );
 ```
+## Transpiled languages
+The template has second-class support for transpiling TypeScript and CoffeeScript.  These are considered second-class and support may be removed in a minor release but not in a patch release.
+
+Overwriting files through the config folder may require you to stick to the original format.  There are currently no guarantees on this.
+
+### Coffeescript
+Any file extending to .coffee will be transpiled from coffeescript to javascript file.  Sourcemaps are included for debugging.
+
+### TypeScript
+Any file extending in .ts will be transpiled to a javascript file.  Sources are currently not typechecked though this is subject to change.  Sourcemaps are included for debugging.
 
 ## Dependencies
 
 You can install additional dependencies by including a `package.json` file next to your `app.js`. It works as you would expect: just define the packages in the `dependencies` section of the `package.json`. They will be installed automatically at build time.
 
 ## Configuration
+### Environment variables
 The following environment variables can be configured:
 
   - `NODE_ENV` (default: `production`): either `"development"` or `"production"`. The environment to start the application in. The application live reloads on changes in `"development"` mode.
   - `MAX_BODY_SIZE` (default: `100kb`): max size of the request body. See [ExpressJS documentation](https://expressjs.com/en/resources/middleware/body-parser.html#limit).
+  - `HOST` (default: `0.0.0.0`): The hostname you want the service to bind to.
+  - `PORT` (default: `80`): The port you want the service to bind to
+
+
+### Mounting `/config`
+You may let users extend the microservice with code.
+
+When you import content from `./config/some-file`, the sources can be provided by the end user in `/config/some-file`.
+
+You may provide default values for each of these files. The sources provided by the app are merged with the sources provided by the microservice, with the app's configuration taking precedence.
 
 ## Logging
 
@@ -177,3 +199,20 @@ When running inside a mu.semte.ch stack, you could mount your sources and connec
         - /absolute/path/to/your/sources/:/app/
 ```
 Now open Chromium, and visit [chrome://inspect/](chrome://inspect/).  Once the service is launched, a remote target on localhost should pop up.
+
+## Handling Delta's
+If you are building a reactive service that should execute certain logic based on changes in the database, you want to hook it up to the [delta-notifier](https://github.com/mu-semtech/delta-notifier/). Some extra steps need to be taken to properly handle delta's, specifically the route handling delta's will need to use a specific bodyParser. 
+
+The default bodyParser provided by the template will only accept `application/vnd.api+json` and the delta-notifier is sending `application/json` content. Aside from that the body of a delta message may be very large, often several megabytes. By specifying the bodyParser on the route accepting delta messages you can easily modify it when required.
+
+An example
+```js 
+// app.js
+import bodyParser from 'body-parser';
+// ...
+
+app.post("/delta-updates", bodyParser.json({limit: '50mb'}), async function(req, res) {
+//...
+}
+```
+

README.md

@@ -0,0 +1,20 @@
+{
+  "presets": [
+    ["@babel/preset-env",
+      {
+        "targets": {
+          "node": 14
+        }
+      }
+    ],
+    ["@babel/preset-typescript"]
+  ],
+  "plugins": [
+    ["@babel/plugin-proposal-decorators", { "legacy": true }],
+    ["@babel/plugin-proposal-class-properties"]
+  ],
+  "ignore": [
+    "./node_modules",
+    "/usr/src/processing/build/node_modules"
+  ]
+}

babel.config.json

@@ -3,25 +3,53 @@ then
     # Run live-reload development
     exec /usr/src/app/node_modules/.bin/nodemon \
          --watch /app \
-         --ext js,mjs,cjs,json \
+         --watch /config \
+         --ext js,coffee,ts,mjs,cjs,json \
          --exec /usr/src/app/run-development.sh
 elif [ "$NODE_ENV" == "production" ]
 then
     diff -rq /app /app.original > /dev/null
-    FILES_CHANGED="$?"
+    APP_FILES_CHANGED="$?"
+    diff -rq /config /config.original > /dev/null
+    CONFIG_FILES_CHANGED="$?"
 
-    if [ ! -f /usr/src/output/app/app.js ]
+    if [ ! -f /usr/src/build/app.js ]
     then
         echo "No built sources found.  If you mount new sources, please set the NODE_ENV=\"development\" environment variable."
         sleep 5;
         exit 1;
-    elif [ $FILES_CHANGED != "0" ]
+    elif [ $APP_FILES_CHANGED != "0" ]
     then
         echo "Built sources are not the same as sources available in /app.  If you mount new sources, please set the NODE_ENV=\"development\" environment variable."
         sleep 5;
         exit 1;
+    elif [ $CONFIG_FILES_CHANGED != "0" ]
+    then
+        echo "Rebuilding sources to include /config."
+
+        # move new configuration into app for transpilation
+        if [[ "$(ls -A /config 2> /dev/null)" ]]
+        then
+            cp -Rf /config/* /usr/src/app/app/config/
+        fi
+
+        # make a backup of the used configuration so we can detect changes
+        rm -Rf /config.original
+        mkdir /config.original
+        if [[ "$(ls -A /config 2> /dev/null)" ]]
+        then
+            cp -Rf /config/* /config.original
+        fi
+
+        # transpile sources
+        cd /usr/src/app/
+        ./transpile-sources.sh
+
+        # boot transpiled sources
+        cd /usr/src/build/
+        exec node ./app.js
     else
-        cd /usr/src/output/
-        exec node ./app/app.js
+        cd /usr/src/build/
+        exec node ./app.js
     fi
 fi

boot.sh

@@ -10,23 +10,24 @@
 cd /usr/src/app
 rm -rf ./app /app.original
 cp -r /app ./
+
+mkdir -p /config /config.original
+
+if [[ "$(ls -A /app/config/ 2> /dev/null)" ]]
+then
+    cp -r /app/config/* /config.original/
+    cp -r /app/config/* /config/
+fi
+
 cp -r /app /app.original
 
 # Install custom packages if need be
 if [ -f ./app/package.json ]
 then
-    npm install ./app
-    rm ./app/package.json
+    echo "Running npm install"
+    cd /usr/src/app/app/
+    npm install
+    cd /usr/src/app/
 fi
 
-# Make the build
-
-## Copy over template node_modules
-mkdir -p /usr/src/output/node_modules
-cp -R /usr/src/app/node_modules /usr/src/output/
-
-## Build microservice sources
-/usr/src/app/node_modules/.bin/babel /usr/src/app/ \
-     --ignore app/node_modules,node_modules \
-     --copy-files --no-copy-ignored \
-     --out-dir /usr/src/output
+./transpile-sources.sh

build-production.sh

@@ -14,6 +14,7 @@ const mu = {
   sparqlEscape: sparql.sparqlEscape,
   sparqlEscapeString: sparql.sparqlEscapeString,
   sparqlEscapeUri: sparql.sparqlEscapeUri,
+  sparqlEscapeDecimal: sparql.sparqlEscapeDecimal,
   sparqlEscapeInt: sparql.sparqlEscapeInt,
   sparqlEscapeFloat: sparql.sparqlEscapeFloat,
   sparqlEscapeDate: sparql.sparqlEscapeDate,
@@ -30,6 +31,7 @@ const SPARQL = mu.SPARQL,
       sparqlEscapeString = mu.sparqlEscapeString,
       sparqlEscapeUri = mu.sparqlEscapeUri,
       sparqlEscapeInt = mu.sparqlEscapeInt,
+      sparqlEscapeDecimal = mu.sparqlEscapeDecimal,
       sparqlEscapeFloat = mu.sparqlEscapeFloat,
       sparqlEscapeDate = mu.sparqlEscapeDate,
       sparqlEscapeDateTime = mu.sparqlEscapeDateTime,
@@ -44,6 +46,7 @@ export {
   sparqlEscape,
   sparqlEscapeString,
   sparqlEscapeUri,
+  sparqlEscapeDecimal,
   sparqlEscapeInt,
   sparqlEscapeFloat,
   sparqlEscapeDate,

helpers/mu/index.js

@@ -4,6 +4,8 @@ import bodyParser from 'body-parser';
 
 var app = express();
 
+var port = process.env.PORT || '80';
+var hostname = process.env.HOST || '0.0.0.0';
 var bodySizeLimit = process.env.MAX_BODY_SIZE || '100kb';
 
 // parse JSONAPI content type
@@ -35,8 +37,8 @@ const errorHandler = function(err, req, res, next) {
 };
 
 // start server
-app.listen( 80, function() {
-  console.log(`Starting server on port 80 in ${app.get('env')} mode`);
+app.listen( port, hostname, function() {
+  console.log(`Starting server on ${hostname}:${port} in ${app.get('env')} mode`);
 });
 
 export default app;