Skip to content

Commit 1030493

Browse files
authored
Merge pull request #1 from blendededge/feature/paging
Feature/paging
2 parents bc48a89 + 9a076d0 commit 1030493

20 files changed

+3993
-1075
lines changed
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# For most projects, this workflow file will not need changing; you simply need
2+
# to commit it to your repository.
3+
#
4+
# You may wish to alter this file to override the set of languages analyzed,
5+
# or to provide custom queries or build logic.
6+
#
7+
# ******** NOTE ********
8+
# We have attempted to detect the languages in your repository. Please check
9+
# the `language` matrix defined below to confirm you have the correct set of
10+
# supported CodeQL languages.
11+
#
12+
name: "CodeQL"
13+
14+
on:
15+
push:
16+
branches: [ master ]
17+
pull_request:
18+
# The branches below must be a subset of the branches above
19+
branches: [ master ]
20+
schedule:
21+
- cron: '34 5 * * 3'
22+
23+
jobs:
24+
analyze:
25+
name: Analyze
26+
runs-on: ubuntu-latest
27+
permissions:
28+
actions: read
29+
contents: read
30+
security-events: write
31+
32+
strategy:
33+
fail-fast: false
34+
matrix:
35+
language: [ 'javascript' ]
36+
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
37+
# Learn more:
38+
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
39+
40+
steps:
41+
- name: Checkout repository
42+
uses: actions/checkout@v2
43+
44+
# Initializes the CodeQL tools for scanning.
45+
- name: Initialize CodeQL
46+
uses: github/codeql-action/init@v1
47+
with:
48+
languages: ${{ matrix.language }}
49+
# If you wish to specify custom queries, you can do so here or in a config file.
50+
# By default, queries listed here will override any specified in a config file.
51+
# Prefix the list here with "+" to use these queries and those in the config file.
52+
# queries: ./path/to/local/query, your-org/your-repo/queries@main
53+
54+
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
55+
# If this step fails, then you should remove it and run the build manually (see below)
56+
- name: Autobuild
57+
uses: github/codeql-action/autobuild@v1
58+
59+
# ℹ️ Command-line programs to run using the OS shell.
60+
# 📚 https://git.io/JvXDl
61+
62+
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
63+
# and modify them (or add more) to build your code if your project
64+
# uses a compiled language
65+
66+
#- run: |
67+
# make bootstrap
68+
# make release
69+
70+
- name: Perform CodeQL Analysis
71+
uses: github/codeql-action/analyze@v1

.github/workflows/master.yml

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: CI to Docker Hub
2+
3+
# Controls when the action will run.
4+
on:
5+
# Triggers the workflow on push or pull request events but only for the master branch
6+
push:
7+
branches: [ master ]
8+
pull_request:
9+
branches: [ master ]
10+
11+
# Allows you to run this workflow manually from the Actions tab
12+
workflow_dispatch:
13+
14+
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
15+
jobs:
16+
# This workflow contains a single job called "build"
17+
build:
18+
# The type of runner that the job will run on
19+
runs-on: ubuntu-latest
20+
21+
# Steps represent a sequence of tasks that will be executed as part of the job
22+
steps:
23+
- name: Check Out Repo
24+
uses: actions/checkout@v2
25+
26+
- name: Set up Docker Buildx
27+
id: buildx
28+
uses: docker/setup-buildx-action@v1
29+
30+
- name: Cache Docker layers
31+
uses: actions/cache@v2
32+
with:
33+
path: /tmp/.buildx-cache
34+
key: ${{ runner.os }}-buildx-${{ github.sha }}
35+
restore-keys: |
36+
${{ runner.os }}-buildx-
37+
38+
- name: Login to Docker Hub
39+
uses: docker/login-action@v1
40+
with:
41+
username: ${{ secrets.DOCKER_HUB_USERNAME }}
42+
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
43+
44+
- name: Build and push
45+
id: docker_build
46+
uses: docker/build-push-action@v2
47+
with:
48+
context: .
49+
builder: ${{ steps.buildx.outputs.name }}
50+
file: ./Dockerfile
51+
push: ${{ github.event_name != 'pull_request' }}
52+
tags: ${{ secrets.DOCKER_REPO_PATH }}/rest-api-component-oih:latest
53+
cache-from: type=local,src=/tmp/.buildx-cache
54+
cache-to: type=local,dest=/tmp/.buildx-cache-new
55+
56+
-
57+
# Temp fix
58+
# https://github.com/docker/build-push-action/issues/252
59+
# https://github.com/moby/buildkit/issues/1896
60+
name: Move cache
61+
run: |
62+
rm -rf /tmp/.buildx-cache
63+
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
64+
65+
- name: Image digest
66+
run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/tests.yml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: Tests
2+
3+
# Controls when the action will run.
4+
on:
5+
# Triggers the workflow on push or pull request events but only for the master branch
6+
push:
7+
branches: [ master ]
8+
pull_request:
9+
branches: [ master ]
10+
11+
# Allows you to run this workflow manually from the Actions tab
12+
workflow_dispatch:
13+
14+
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
15+
jobs:
16+
# This workflow contains a single job called "build"
17+
unit-tests:
18+
# The type of runner that the job will run on
19+
runs-on: ubuntu-latest
20+
21+
# Steps represent a sequence of tasks that will be executed as part of the job
22+
steps:
23+
- name: Check Out Repo
24+
uses: actions/checkout@v2
25+
26+
- name: Set up node
27+
uses: actions/setup-node@v2
28+
with:
29+
node-version: 12.x
30+
31+
- name: use cache
32+
uses: actions/cache@v2
33+
with:
34+
path: |
35+
node_modules
36+
*/*/node_modules
37+
key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
38+
39+
- name: Install dependencies
40+
run: yarn install
41+
42+
- name: Run unit tests
43+
run: yarn test

CHANGELOG.md

Lines changed: 0 additions & 66 deletions
This file was deleted.

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM node:10-alpine AS base
1+
FROM node:12-alpine AS base
22
RUN apk --no-cache add \
33
python \
44
make \

README.md

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,19 @@ _Numbers show: (1) The URL and method of the REST API resource, (2) the HTTP cal
4141
- 504: Gateway Timeout
4242
- DNS lookup timeout
4343

44-
4. `Do not verify SSL certificate (unsafe)` - disable verifying the server certificate - **unsafe**.
45-
5. `Follow redirect mode` - If you want disable Follow Redirect functionality, you can use option `Follow redirect mode`.By default `Follow redirect mode` option has value `Follow redirects`.
46-
6. `Delay` - If you want to slow down requests to your API you can set delay value (in seconds) and the component will delay calling the next request after the previous request.
44+
- `Do not verify SSL certificate (unsafe)` - disable verifying the server certificate - **unsafe**.
45+
- `Follow redirect mode` - If you want disable Follow Redirect functionality, you can use option `Follow redirect mode`.By default `Follow redirect mode` option has value `Follow redirects`.
46+
- `Delay` - If you want to slow down requests to your API you can set delay value (in seconds) and the component will delay calling the next request after the previous request.
4747
Time for the delay is calculated as `Delay`/ `Call Count` and shouldn't be more than 1140 seconds (19 minutes due to platform limitation).
4848
The `Call Count` value by default is 1. If you want to use another value, please set the `Call Count` field.
4949
Notice: See [Known Limitations](#known-limitations) about `Delay` value.
50-
7. `Call Count` - the field should be used only in pair with `Delay`, default to 1.
51-
8. `Request timeout` - Timeout period in milliseconds (1-1140000) while component waiting for server response, also can be configured with REQUEST_TIMEOUT environment variable if configuration field is not provided. Defaults to 100000 (100 sec).
50+
- `Call Count` - the field should be used only in pair with `Delay`, default to 1.
51+
- `Request timeout` - Timeout period in milliseconds (1-1140000) while component waiting for server response, also can be configured with REQUEST_TIMEOUT environment variable if configuration field is not provided. Defaults to 100000 (100 sec).
52+
5253
Notice: Specified for component REQUEST_TIMEOUT enviroment variable would be overwritten by specified value of Request timeout, default value would be also overwritten
54+
- `responseToSnapshotTransform` - This is a JSONata applied to the REST response body and stored in the snapshot as an object.
55+
56+
4. `Snapshot in URL tranform` - If a snapshot value is available it is added into the msg.data object as `msg.data.oihsnapshot`. This can be used in conjuction with the `responseToSnapshotTransform` to perform paging. You can save information for the next page from the response in the snapshot and then use the snapshot information in the next request URL.
5357

5458
## Authorisation methods
5559

lib/AttachmentProcessor.js

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
const axios = require('axios');
2+
const uuid = require('uuid');
3+
4+
const REQUEST_TIMEOUT = process.env.REQUEST_TIMEOUT ? parseInt(process.env.REQUEST_TIMEOUT, 10) : 10000; // 10s
5+
const REQUEST_MAX_RETRY = process.env.REQUEST_MAX_RETRY ? parseInt(process.env.REQUEST_MAX_RETRY, 10) : 7; // 10s
6+
const REQUEST_RETRY_DELAY = process.env.REQUEST_RETRY_DELAY ? parseInt(process.env.REQUEST_RETRY_DELAY, 10) : 7000; // 7s
7+
const REQUEST_MAX_CONTENT_LENGTH = process.env.REQUEST_MAX_CONTENT_LENGTH ? parseInt(process.env.REQUEST_MAX_CONTENT_LENGTH, 10) : 10485760; // 10MB
8+
const { ATTACHMENT_STORAGE_SERVICE_BASE_URL } = process.env;
9+
10+
// Adapted from https://github.com/elasticio/component-commons-library/blob/master/lib/attachment/AttachmentProcessor.ts
11+
class AttachmentProcessor {
12+
constructor(emitter, token, attachmentStorageServiceBaseUrl) {
13+
this.attachmentService = attachmentStorageServiceBaseUrl;
14+
this.emitter = emitter;
15+
this.token = token;
16+
}
17+
18+
async getAttachment(url, responseType) {
19+
const ax = axios.create();
20+
AttachmentProcessor.addRetryCountInterceptorToAxios(ax);
21+
22+
console.log(`Getting attachment ${responseType} from ${url}`);
23+
const axConfig = {
24+
url,
25+
responseType,
26+
method: 'get',
27+
timeout: REQUEST_TIMEOUT,
28+
retry: REQUEST_MAX_RETRY,
29+
delay: REQUEST_RETRY_DELAY,
30+
withCredentials: true,
31+
headers: {
32+
Authorization: `Bearer ${this.token}`,
33+
},
34+
};
35+
36+
return ax(axConfig);
37+
}
38+
39+
async uploadAttachment(body, mimeType) {
40+
const putUrl = await AttachmentProcessor.preparePutUrl(this.attachmentService);
41+
const ax = axios.create();
42+
AttachmentProcessor.addRetryCountInterceptorToAxios(ax);
43+
44+
const axConfig = {
45+
url: putUrl,
46+
data: body,
47+
method: 'put',
48+
timeout: REQUEST_TIMEOUT,
49+
retry: REQUEST_MAX_RETRY,
50+
delay: REQUEST_RETRY_DELAY,
51+
maxContentLength: REQUEST_MAX_CONTENT_LENGTH,
52+
withCredentials: true,
53+
headers: {
54+
Authorization: `Bearer ${this.token}`,
55+
'Content-Type': mimeType,
56+
},
57+
};
58+
59+
return ax(axConfig);
60+
}
61+
62+
static async preparePutUrl(attachmentService) {
63+
const service = attachmentService || ATTACHMENT_STORAGE_SERVICE_BASE_URL;
64+
const signedUrl = `${service}/objects/${uuid.v4()}`;
65+
66+
this.emitter.logger.debug(`Attachment Storage Service signed url is ${signedUrl}`);
67+
return signedUrl;
68+
}
69+
70+
static addRetryCountInterceptorToAxios(ax) {
71+
ax.interceptors.response.use(undefined, (err) => { // Retry count interceptor for axios
72+
const { config } = err;
73+
if (!config || !config.retry || !config.delay) {
74+
return Promise.reject(err);
75+
}
76+
config.currentRetryCount = config.currentRetryCount || 0;
77+
if (config.currentRetryCount >= config.retry) {
78+
return Promise.reject(err);
79+
}
80+
config.currentRetryCount += 1;
81+
return new Promise(resolve => setTimeout(() => resolve(ax(config)), config.delay));
82+
});
83+
}
84+
}
85+
exports.AttachmentProcessor = AttachmentProcessor;

lib/actions/httpRequestAction.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
const { processMethod } = require("../utils.js");
1+
const { processMethod } = require('../utils.js');
22

33
function processAction(msg, cfg) {
4-
console.log("msg: ", msg);
5-
console.log("cfg: ", cfg);
4+
this.logger.debug('msg: ', msg);
5+
this.logger.debug('cfg: ', cfg);
66

77
return processMethod.call(this, msg, cfg);
88
}

lib/attachments.js

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
function addAttachment(msg, name, data, contentLength, contentType, sourceUrl, responseType) {
2+
const emitter = this;
3+
switch (responseType) {
4+
case 'arraybuffer':
5+
break;
6+
case 'stream':
7+
break;
8+
default:
9+
throw Error('Please set the response type to arraybuffer or stream to handle the binary file attached to the response.');
10+
}
11+
}

0 commit comments

Comments
 (0)