From e960b5482f17106d1cb6c2da97375c281d29a083 Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 21 Mar 2022 15:05:22 -0700 Subject: [PATCH] EPOC-540 Added lint or custom DNS feature --- package.json | 2 +- src/configuration/config.js | 2 + src/external/essRequests/socketHelper.js | 8 ++- src/external/mdeployRequests.js | 7 +- src/external/messRequests.js | 3 +- src/index.js | 4 +- src/lib/anaxHelper/essHelper.js | 13 ++-- src/lib/anaxHelper/index.js | 2 +- src/lib/anaxHelper/scripts.js | 48 ++++++------- src/lib/jobs/gatewayNodeSyncJob.js | 3 - src/lib/mdeployCleanup.js | 6 +- src/lib/scriptHelper.js | 1 - src/lib/startHelper.js | 6 +- src/scripts/anax.sh | 87 +++++++++++++----------- src/util/nodeUtil.js | 1 - test/ess.js | 2 +- 16 files changed, 98 insertions(+), 97 deletions(-) diff --git a/package.json b/package.json index 750830c..400e5de 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mimik_ohorchestrator", - "version": "1.1.2", + "version": "1.1.3", "swaggerFile": { "version": "1.1.0", "account": "mimik", diff --git a/src/configuration/config.js b/src/configuration/config.js index c303c2e..6492e4e 100644 --- a/src/configuration/config.js +++ b/src/configuration/config.js @@ -41,6 +41,7 @@ const pack = require('../../package.json'); * | EDGE_DEPLOYMENT_CONTAINER_ENV | edgeEngine container env representing edge deployment | HZN_DEPLOYMENT_LOCATION=edgeNode | * | ANAX_DOCKER_TAG | Anax container version | 2.30.0-794 | * | MAXIMUM_FILE_SIZE | Maximum file size that can be shared using content delivery (ESS/mess) | 100 | in MB + * | ANAX_CONTAINERS_CUSTOM_DNS_IP | Ip address of the DNS server for anax docker containers | 'empty string' | Not required * * These values are on top of what is needed in the [configuration](https://bitbucket.org/mimiktech/configuration) library. * @@ -109,6 +110,7 @@ module.exports = (() => { gatewayNodeSyncJobInterval: parseInt(process.env.GATEWAY_NODE_SYNC_JOB_INTERVAL, 10) || 120, anaxContainersPortNumStart: parseInt(process.env.ANAX_CONTAINERS_PORT_NUM_START, 10) || 8200, anaxContainersPortNumEnd: parseInt(process.env.ANAX_CONTAINERS_PORT_NUM_END, 10) || 8999, + anaxContainersCustomDNSIP: process.env.ANAX_CONTAINERS_CUSTOM_DNS_IP || '', edgeDeploymentContainerEnv: process.env.EDGE_DEPLOYMENT_CONTAINER_ENV || 'HZN_DEPLOYMENT_LOCATION=edgeNode', }, }); diff --git a/src/external/essRequests/socketHelper.js b/src/external/essRequests/socketHelper.js index 4431a23..c47b704 100644 --- a/src/external/essRequests/socketHelper.js +++ b/src/external/essRequests/socketHelper.js @@ -43,7 +43,8 @@ const dataRequest = (nodeId, request, correlationId) => new Promise((resolve, re const clientRequest = https.request(request, callback); if (request.body) clientRequest.write(request.body); clientRequest.end(); - } catch (error) { + } + catch (error) { reject(error); } }); @@ -66,8 +67,9 @@ const fileDownloadRequest = (nodeId, outputFilePath, request, correlationId) => try { const clientRequest = https.request(request, callback); if (request.body) clientRequest.write(request.body); - clientRequest.end(); - } catch (error) { + clientRequest.end(); + } + catch (error) { reject(error); } }); diff --git a/src/external/mdeployRequests.js b/src/external/mdeployRequests.js index 327bc56..3f5da97 100644 --- a/src/external/mdeployRequests.js +++ b/src/external/mdeployRequests.js @@ -87,9 +87,9 @@ const getContainersForExternalNode = (externalNodeId, correlationId) => rp({ } const { data } = response.data[0].responseBody; return data; - }) + }); -const deleteContainersForExternalNode = (externalNodeId, containerIds, correlationId) => Promise.map(containerIds, (containerId) => rp({ +const deleteContainersForExternalNode = (externalNodeId, containerIds, correlationId) => Promise.map(containerIds, () => rp({ method: 'POST', headers: { 'x-correlation-id': correlationId, @@ -103,8 +103,7 @@ const deleteContainersForExternalNode = (externalNodeId, containerIds, correlati method: 'DELETE', }, }, -}) -.catch(() => {})); +}).catch(() => {})); const clientStatusValues = { ACTIVE: 'active', diff --git a/src/external/messRequests.js b/src/external/messRequests.js index 88dc504..2770af2 100644 --- a/src/external/messRequests.js +++ b/src/external/messRequests.js @@ -35,7 +35,6 @@ const postFile = (nodeId, pathName, fileName, localFilePath, correlationId) => r const form = new FormData(); form.append('file', fs.createReadStream(localFilePath)); - return rpRetry({ method: 'PUT', headers: { @@ -47,7 +46,7 @@ const postFile = (nodeId, pathName, fileName, localFilePath, correlationId) => r data: form, maxBodyLength: maxFileSize, maxContentLength: maxFileSize, - }) + }); }); module.exports = { diff --git a/src/index.js b/src/index.js index ea4cd95..7ca31d5 100644 --- a/src/index.js +++ b/src/index.js @@ -9,7 +9,7 @@ const { startJobs } = require('./lib/jobs'); const { startupTasks } = require('./lib/startHelper'); const { cleanupAllNodes } = require('./lib/mdeployCleanup'); -let correlationId = getCorrelationId('service-startup-preOps'); +const correlationId = getCorrelationId('service-startup-preOps'); init(app, __dirname, config, [], cluster(config), { preOps: [ @@ -23,6 +23,4 @@ init(app, __dirname, config, [], cluster(config), { ({ config } = result); }); - - module.exports = app; diff --git a/src/lib/anaxHelper/essHelper.js b/src/lib/anaxHelper/essHelper.js index c01f2ee..2a2adda 100644 --- a/src/lib/anaxHelper/essHelper.js +++ b/src/lib/anaxHelper/essHelper.js @@ -1,3 +1,4 @@ +/* eslint-disable consistent-return */ const fs = require('fs-extra'); const logger = require('@mimik/sumologic-winston-logger'); @@ -31,8 +32,7 @@ const { } = require('../../configuration/config'); const processingObjects = {}; -const ESS_OBJECT_DOWNLOADED_STATUS = 'completelyReceived'; -// const ESS_OBJECT_RECEIVED_STATUS = 'objreceived'; TODO remove if not used +const ESS_OBJECT_DOWNLOADED_STATUS = 'completelyReceived'; // 'objreceived' is the final received status ie when received by oh-orchestrator and marked by API const getObjectsBeingDownloadedFlag = (nodeId, agreementId, objectType, objectId) => `${nodeId}_${agreementId}_${objectType}_${objectId}`; @@ -59,8 +59,8 @@ const pollForObjectByType = (nodeId, agreementId, objectType, correlationId) => const contentDeploymentInstructionProperty = objectProperties.find( (property) => property.type === gatewayDeploymentPropertyType - && property.name === gatewayDeploymentPropertyName - ) + && property.name === gatewayDeploymentPropertyName, + ); if (!contentDeploymentInstructionProperty) return; @@ -72,6 +72,7 @@ const pollForObjectByType = (nodeId, agreementId, objectType, correlationId) => processingObjects[objectsBeingDownloadedFlag] = true; + // eslint-disable-next-line consistent-return return getCurrentNode(correlationId) .then((gatewayNode) => { const outputFileDir = getESSStorageDir(nodeId, agreementId, objectType); @@ -89,15 +90,13 @@ const pollForObjectByType = (nodeId, agreementId, objectType, correlationId) => }) .finally(() => { delete processingObjects[objectsBeingDownloadedFlag]; - }) + }); }); }); }) .catch(() => { }); }); - - const initializePolling = (node, correlationId) => { const { id: nodeId } = node; diff --git a/src/lib/anaxHelper/index.js b/src/lib/anaxHelper/index.js index d329c23..1acc97b 100644 --- a/src/lib/anaxHelper/index.js +++ b/src/lib/anaxHelper/index.js @@ -2,7 +2,7 @@ const Promise = require('bluebird'); const getPort = require('get-port'); const { updateAnaxState } = require('../../models/nodeModel'); -const { initializePolling } =require('./essHelper'); +const { initializePolling } = require('./essHelper'); const { findNode, createNode, diff --git a/src/lib/anaxHelper/scripts.js b/src/lib/anaxHelper/scripts.js index 89e9c75..b796966 100644 --- a/src/lib/anaxHelper/scripts.js +++ b/src/lib/anaxHelper/scripts.js @@ -2,10 +2,10 @@ const Promise = require('bluebird'); const fs = require('fs-extra'); const logger = require('@mimik/sumologic-winston-logger'); -const { rpRetry } = require('@mimik/request-retry'); const { getRichError } = require('@mimik/response-helper'); const { scriptFileValues } = require('../../util/scriptUtil'); +const { anaxContainersCustomDNSIP } = require('../../configuration/config'); const { getArch, scriptCommandValues } = require('../../util/anaxUtil'); const { checkIfNodeConfigured } = require('../../external/anaxRequests'); const { runScriptFile, runScriptCommand } = require('../scriptHelper'); @@ -28,10 +28,11 @@ const { getNodeConfigFilePath, } = require('../../util/anaxUtil'); -const timeout = () => new Promise((resolve, reject) => { setTimeout(resolve, 5000); }); +const timeout = () => new Promise((resolve) => { setTimeout(resolve, 5000); }); const updateHznCliConfig = (nodeId, correlationId) => { const configFilePath = getNodeConfigFilePath(nodeId); + // eslint-disable-next-line max-len const configFileData = `HZN_EXCHANGE_URL=${exchangeUrl}\nHZN_FSS_CSSURL=${cssUrl}\nHZN_NODE_ID=${nodeId}\nHZN_ORG_ID=${orgId}\nHZN_EXCHANGE_USER_AUTH=${exchangeUserAuth}\nHZN_AGBOT_URL=${agbotUrl}\n`; return fs.ensureFile(configFilePath) @@ -77,6 +78,7 @@ const deployAnaxNode = (nodeId, nodePort, dockerSocketFilePath, correlationId) = HORIZON_AGENT_PORT: nodePort, ANAX_TAG: anaxDockerTag, ARCH: getArch(), + CUSTOM_DNS_IP: anaxContainersCustomDNSIP === '' ? undefined : anaxContainersCustomDNSIP, }, correlationId, ]; @@ -120,29 +122,27 @@ const registerAnaxNode = (nodeId, nodePort, policyFilePath, correlationId) => { }); }; -const unregisterAnaxNode = (nodeId, nodePort, correlationId) => { - return runScriptCommand( - scriptCommandValues.UNREGISTER_ANAX, - undefined, - { - HZN_ORG_ID: orgId, - HORIZON_URL: `http://localhost:${nodePort}`, - HZN_EXCHANGE_URL: exchangeUrl, - HZN_EXCHANGE_USER_AUTH: exchangeUserAuth, - HZN_EXCHANGE_NODE_AUTH: `${nodeId}:${defaultNodeToken}`, - }, - correlationId, - ) - .then(timeout) - .then(() => checkIfNodeConfigured(nodePort, correlationId)) - .then(() => { - if (isNodeConfigured) throw getRichError('System', 'Node did not get unregistered', { nodeId, nodePort, policyFilePath }, null, 'error', correlationId); - - logger.debug('Unregistered Anax Node', { - nodeId, nodePort, correlationId, - }); +const unregisterAnaxNode = (nodeId, nodePort, correlationId) => runScriptCommand( + scriptCommandValues.UNREGISTER_ANAX, + undefined, + { + HZN_ORG_ID: orgId, + HORIZON_URL: `http://localhost:${nodePort}`, + HZN_EXCHANGE_URL: exchangeUrl, + HZN_EXCHANGE_USER_AUTH: exchangeUserAuth, + HZN_EXCHANGE_NODE_AUTH: `${nodeId}:${defaultNodeToken}`, + }, + correlationId, +) + .then(timeout) + .then(() => checkIfNodeConfigured(nodePort, correlationId)) + .then((isNodeConfigured) => { + if (isNodeConfigured) throw getRichError('System', 'Node did not get unregistered', { nodeId, nodePort }, null, 'error', correlationId); + + logger.debug('Unregistered Anax Node', { + nodeId, nodePort, correlationId, }); -}; + }); const purgeDocker = () => runScriptCommand(scriptCommandValues.NUKE_DOCKER) .catch(() => { }); // To catch if no containers found diff --git a/src/lib/jobs/gatewayNodeSyncJob.js b/src/lib/jobs/gatewayNodeSyncJob.js index cd4abdd..a7bbb3b 100644 --- a/src/lib/jobs/gatewayNodeSyncJob.js +++ b/src/lib/jobs/gatewayNodeSyncJob.js @@ -1,5 +1,3 @@ -const uuid = require('uuid'); - const logger = require('@mimik/sumologic-winston-logger'); const { getRichError } = require('@mimik/response-helper'); const { getCorrelationId } = require('@mimik/request-helper'); @@ -55,7 +53,6 @@ const start = (correlationId) => getClient(correlationId) interval = setInterval(syncNodes, gatewayNodeSyncJobInterval * 1000); }); - const stop = () => Promise.resolve() .then(() => { clearInterval(interval); diff --git a/src/lib/mdeployCleanup.js b/src/lib/mdeployCleanup.js index ce33597..f87b4e1 100644 --- a/src/lib/mdeployCleanup.js +++ b/src/lib/mdeployCleanup.js @@ -18,10 +18,8 @@ const cleanupNode = (id, correlationId) => getContainersForExternalNode(id, corr // TODO Handle error .catch(() => { }); -const cleanupAllNodes = (correlationId) => { - return nodeModel.getAllNodes(correlationId) - .then((nodes) => Promise.map(nodes, ({ id }) => cleanupNode(id, correlationId))); -}; +const cleanupAllNodes = (correlationId) => nodeModel.getAllNodes(correlationId) + .then((nodes) => Promise.map(nodes, ({ id }) => cleanupNode(id, correlationId))); module.exports = { cleanupNode, diff --git a/src/lib/scriptHelper.js b/src/lib/scriptHelper.js index a3653db..b126b01 100644 --- a/src/lib/scriptHelper.js +++ b/src/lib/scriptHelper.js @@ -2,7 +2,6 @@ const Promise = require('bluebird'); const childProcess = require('child_process'); const logger = require('@mimik/sumologic-winston-logger'); -const { getRichError } = require('@mimik/response-helper'); const runScriptFile = (scriptFileName, args = '', env = {}, correlationId) => { let scriptEnvs = ''; diff --git a/src/lib/startHelper.js b/src/lib/startHelper.js index 3f33819..7385a6c 100644 --- a/src/lib/startHelper.js +++ b/src/lib/startHelper.js @@ -17,10 +17,8 @@ const startupTasks = (newCorrelationId) => { correlationId = newCorrelationId; return makeScriptExecutable(); -} - - +}; module.exports = { startupTasks, -} \ No newline at end of file +}; diff --git a/src/scripts/anax.sh b/src/scripts/anax.sh index 655e997..f2e9471 100755 --- a/src/scripts/anax.sh +++ b/src/scripts/anax.sh @@ -13,7 +13,11 @@ # export ESS_SOCKET_DIR="${HOST_SHARE_PATH}/essSocket" # export CONTAINER_LABEL=containerType=anaxNode # -### Required envs in config file +# Optional envs +# +# export CUSTOM_DNS_IP= +# +### Required params in node config file # # HZN_EXCHANGE_URL=http://192.168.1.115:3090/v1/ # HZN_FSS_CSSURL=http://192.168.1.115:9443 @@ -25,7 +29,7 @@ ### Required for hzn cli # # export HORIZON_URL=http://localhost:8200 -# export HZN_EXCHANGE_URL=http://192.168.1.115:3090/v1/ +# export HZN_EXCHANGE_URL=http://192.168.1.115:3090/v1/ # export HZN_EXCHANGE_USER_AUTH=admin:password # export HZN_ORG_ID=myorg # export HZN_EXCHANGE_NODE_AUTH=bc8ebd2abf839833:nodeToken @@ -37,59 +41,66 @@ ANAX_IMAGE=${ANAX_IMAGE:-openhorizon/${ARCH}_anax} ANAX_TAG=${ANAX_TAG:-2.30.0-794} CONTAINER_LABEL=${CONTAINER_LABEL:-containerType=anaxNode} +ADDITIONAL_FLAGS=${ADDITIONAL_FLAGS} + validateStart() { - [[ -z "${HZN_NODE_ID}" ]] && echo "HZN_NODE_ID is required env" 1>&2 && exit 1 - [[ -z "${HORIZON_AGENT_PORT}" ]] && echo "HORIZON_AGENT_PORT is required env" 1>&2 && exit 1 - [[ -z "${ESS_AUTH_DIR}" ]] && echo "ESS_AUTH_DIR is required env" 1>&2 && exit 1 - [[ -z "${ESS_SOCKET_DIR}" ]] && echo "ESS_SOCKET_DIR is required env" 1>&2 && exit 1 + [[ -z "${HZN_NODE_ID}" ]] && echo "HZN_NODE_ID is required env" 1>&2 && exit 1 + [[ -z "${HORIZON_AGENT_PORT}" ]] && echo "HORIZON_AGENT_PORT is required env" 1>&2 && exit 1 + [[ -z "${ESS_AUTH_DIR}" ]] && echo "ESS_AUTH_DIR is required env" 1>&2 && exit 1 + [[ -z "${ESS_SOCKET_DIR}" ]] && echo "ESS_SOCKET_DIR is required env" 1>&2 && exit 1 + + [[ ! -f "${CONFIG_PATH}" ]] && echo "CONFIG_PATH: $CONFIG_PATH file does not exist" 1>&2 && exit 1 + [[ ! -S "${DOCKER_SOCKET}" ]] && echo "DOCKER_SOCKET: $DOCKER_SOCKET file does not exist" 1>&2 && exit 1 - [[ ! -f "${CONFIG_PATH}" ]] && echo "CONFIG_PATH: $CONFIG_PATH file does not exist" 1>&2 && exit 1 - [[ ! -S "${DOCKER_SOCKET}" ]] && echo "DOCKER_SOCKET: $DOCKER_SOCKET file does not exist" 1>&2 && exit 1 + [[ ! -z "${CUSTOM_DNS_IP}" ]]; then + ADDITIONAL_FLAGS="--dns=${CUSTOM_DNS_IP}" + fi } validateStop() { - [[ -z "${HZN_NODE_ID}" ]] && echo "HZN_NODE_ID is required env" 1>&2 && exit 1 + [[ -z "${HZN_NODE_ID}" ]] && echo "HZN_NODE_ID is required env" 1>&2 && exit 1 } start() { - validateStart - -docker run -d -t --restart always --name $DOCKER_NAME --privileged \ --p 127.0.0.1:${HORIZON_AGENT_PORT}:8510 -e DOCKER_NAME=${DOCKER_NAME} \ --e HZN_VAR_RUN_BASE=/var/tmp/horizon/${DOCKER_NAME} -l ${CONTAINER_LABEL} \ --v ${DOCKER_SOCKET}:/var/run/docker.sock -v ${CONFIG_PATH}:/etc/default/horizon \ --v ${ESS_AUTH_DIR}:/var/horizon/ess-auth -v ${ESS_SOCKET_DIR}:/var/tmp/horizon/${DOCKER_NAME} \ -${ANAX_IMAGE}:${ANAX_TAG} - -# -v ${DOCKER_NAME}_etc:/etc/horizon/ -v ${DOCKER_NAME}_var:/var/horizon/ \ Mappings not required + validateStart + + docker run -d -t --restart always --name $DOCKER_NAME --privileged \ + ADDITIONAL_FLAGS \ + -p 127.0.0.1:${HORIZON_AGENT_PORT}:8510 -e DOCKER_NAME=${DOCKER_NAME} \ + -e HZN_VAR_RUN_BASE=/var/tmp/horizon/${DOCKER_NAME} -l ${CONTAINER_LABEL} \ + -v ${DOCKER_SOCKET}:/var/run/docker.sock -v ${CONFIG_PATH}:/etc/default/horizon \ + -v ${ESS_AUTH_DIR}:/var/horizon/ess-auth -v ${ESS_SOCKET_DIR}:/var/tmp/horizon/${DOCKER_NAME} \ + ${ANAX_IMAGE}:${ANAX_TAG} + + # -v ${DOCKER_NAME}_etc:/etc/horizon/ -v ${DOCKER_NAME}_var:/var/horizon/ \ Mappings not required } stop() { - validateStop - - docker stop -t 60 $DOCKER_NAME - docker rm -f $DOCKER_NAME - docker volume rm ${DOCKER_NAME}_var ${DOCKER_NAME}_etc + validateStop + + docker stop -t 60 $DOCKER_NAME + docker rm -f $DOCKER_NAME + docker volume rm ${DOCKER_NAME}_var ${DOCKER_NAME}_etc } restart() { - stop - start + stop + start } case "$1" in - start) - start - ;; - stop) - stop - ;; - restart|update) - restart - ;; - *) - echo "Unknown value: use either start, stop, restart/update with ./anax.sh" - exit 1; + start) + start + ;; + stop) + stop + ;; + restart|update) + restart + ;; + *) + echo "Unknown value: use either start, stop, restart/update with ./anax.sh" + exit 1; esac exit \ No newline at end of file diff --git a/src/util/nodeUtil.js b/src/util/nodeUtil.js index 8cd883e..8949590 100644 --- a/src/util/nodeUtil.js +++ b/src/util/nodeUtil.js @@ -16,7 +16,6 @@ const gatewayNodeIds = { const gatewayNodeIdsPortsMap = {}; gatewayNodeIdsPortsMap[gatewayNodeIds.DOCKER] = 8071; - module.exports = { gatewayNodeIds, gatewayNodeIdsPortsMap, diff --git a/test/ess.js b/test/ess.js index 93dfeaf..b7eb0e9 100644 --- a/test/ess.js +++ b/test/ess.js @@ -7,7 +7,7 @@ const { } = require('../src/external/essRequests'); process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; -process.env.LOG_MODE="none"; +process.env.LOG_MODE = 'none'; const nodeId = '89b5345b4a3d01ed5d053417e96bd9df320e765adb12dd9971485dc5'; const agreementId = '6bec1300d84a95d0acca516f1855dd22fe8ec17bb36a165683ae2777e18baebf';