Skip to content

Commit

Permalink
Replaces Logstash + redis with couch2pg++ (#109)
Browse files Browse the repository at this point in the history
Removes Logstash
Removes redis
Removes redis worker
Switches e2e tests from jest to mocha to conform with all of our other repos
Adds a refactored version of Couch2pg. 

Remaining todos: 
- watch changes after initial transfer has finished
- add support for multiple databases
- figure out what we do with deletes
  • Loading branch information
dianabarsan authored Jun 7, 2024
1 parent 16666be commit c4c314c
Show file tree
Hide file tree
Showing 47 changed files with 3,996 additions and 14,343 deletions.
93 changes: 57 additions & 36 deletions .eslintrc.json
Original file line number Diff line number Diff line change
@@ -1,38 +1,59 @@
{
"env": {
"es2021": true,
"node": true
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended"
],
"overrides": [
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": "latest",
"sourceType": "module"
},
"plugins": [
"@typescript-eslint"
],
"rules": {
"indent": [
"error",
"space"
],
"linebreak-style": [
"error",
"unix"
],
"quotes": [
"error",
"double"
],
"semi": [
"error",
"always"
]
}
"env": {
"es2021": true,
"node": true
},
"extends": [
"@medic",
"plugin:node/recommended"
],
"overrides": [
],
"parserOptions": {
"ecmaVersion": "latest",
"sourceType": "module"
},
"plugins": [
"node"
],
"rules": {
"array-bracket-newline": ["error", "consistent"],
"array-callback-return": ["error", { "allowImplicit": true }],
"arrow-spacing": ["error", { "before": true, "after": true }],
"brace-style": ["error", "1tbs"],
"comma-spacing": ["error", { "before": false, "after": true }],
"comma-style": ["error", "last"],
"default-param-last": "error",
"dot-location": ["error", "property"],
"dot-notation": ["error", { "allowKeywords": true }],
"func-call-spacing": ["error", "never"],
"func-style": ["error", "expression"],
"function-call-argument-newline": ["error", "consistent"],
"function-paren-newline": ["error", "consistent"],
"implicit-arrow-linebreak": ["error", "beside"],
"key-spacing": ["error", { "beforeColon": false, "afterColon": true }],
"keyword-spacing": ["error", { "before": true, "after": true }],
"linebreak-style": ["error", "unix"],
"lines-between-class-members": ["error", "always", { "exceptAfterSingleLine": true }],
"new-parens": "error",
"no-alert": "error",
"no-else-return": "error",
"no-extra-bind": "error",
"no-console": "off",
"no-lone-blocks": "error",
"no-nested-ternary": "error",
"no-undef-init": "error",
"no-useless-rename": "error",
"no-whitespace-before-property": "error",
"node/no-exports-assign": "error",
"node/no-unsupported-features/es-syntax": "off",
"rest-spread-spacing": ["error", "never"],
"semi-spacing": ["error", { "before": false, "after": true }],
"semi-style": ["error", "last"],
"template-curly-spacing": "error",
"unicode-bom": ["error", "never"],
"node/no-extraneous-import": ["error", {
"allowModules": ["chai", "chai-as-promised", "sinon", "esmock"]
}]
}
}
16 changes: 4 additions & 12 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Build and push logstash
- name: Build and push couch2pg
uses: docker/build-push-action@v4
with:
context: ./logstash
file: ./logstash/Dockerfile
context: ./couch2pg
file: ./couch2pg/Dockerfile
push: true
tags: medicmobile/cht-logstash-couchdb:latest
tags: medicmobile/cht-sync-couch2pg:latest

- name: Build and push dataemon
uses: docker/build-push-action@v4
Expand All @@ -33,11 +33,3 @@ jobs:
file: ./dbt/Dockerfile
push: true
tags: medicmobile/dataemon:latest

- name: Build and push redis-worker
uses: docker/build-push-action@v4
with:
context: ./redis-worker
file: ./redis-worker/Dockerfile
push: true
tags: dockermedic/cht-sync-redis-worker:latest
64 changes: 21 additions & 43 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,37 +3,28 @@ name: Test
on: [push, pull_request]

jobs:
unit-tests:
name: Unit tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: 20.x
- run: npm ci
- run: npm ci
working-directory: ./couch2pg
- run: npm run lint
- run: npm run test

e2e-tests:
name: E2E Tests
runs-on: ubuntu-latest
env:
POSTGRES_USER: "postgres"
POSTGRES_PASSWORD: "postgres"
POSTGRES_DB: "data"
POSTGRES_TABLE: "medic"
POSTGRES_SCHEMA: "v1"
DBT_POSTGRES_USER: "postgres"
DBT_POSTGRES_PASSWORD: "postgres"
DBT_POSTGRES_SCHEMA: "dbt"
DBT_POSTGRES_HOST: "postgres"
CHT_PIPELINE_BRANCH_URL: "https://github.com/medic/cht-pipeline.git#main"
COUCHDB_USER: "medic"
COUCHDB_PASSWORD: "password"
COUCHDB_DBS: "medic"
COUCHDB_HOST: "couchdb"
COUCHDB_PORT: 5984
COUCHDB_SECURE: false
REDIS_HOST: redis
REDIS_PORT: 6379
REDIS_KEY: medic
POSTGREST_ENDPOINT: postgrest:3000
REDIS_BATCH_SIZE: 100
POSTGRES_HOST: postgres
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: 16.x
node-version: 20.x
- name: Get Docker Hub username
id: get-docker-hub-username
run: echo '::set-output name=dockerhub_username::${{ secrets.DOCKERHUB_USERNAME }}'
Expand All @@ -43,30 +34,17 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
if: steps.get-docker-hub-username.outputs.dockerhub_username
- name: Install
run: npm install
- name: Run local
- run: npm ci
- name: Start containers
run: >
docker-compose
docker compose
--env-file ./tests/.e2e-env
-f docker-compose.yml
-f docker-compose.couchdb.yml
-f docker-compose.postgres.yml
-f docker-compose.yml
up -d
- name: Sleep for 60 seconds
run: sleep 60s
- name: Run e2e tests
run: npm run test:e2e
worker-tests:
name: Worker Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: 16.x
- name: Install
working-directory: ./redis-worker
run: npm install
- name: Run worker tests
working-directory: ./redis-worker
run: npm test

3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@
node_modules
.idea
*/coverage/*
logstash/sequence_path.txt
/couch2pg/.nyc_output/
/.eslintcache
2 changes: 2 additions & 0 deletions .npmignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
/tests/
/couch2pg/tests/
9 changes: 9 additions & 0 deletions couch2pg/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
FROM node:20-alpine AS base_build

WORKDIR /

COPY package.json package-lock.json ./
COPY ./src ./src
RUN npm ci --ignore-scripts

CMD ["node", "src/index.js"]
Loading

0 comments on commit c4c314c

Please sign in to comment.