From 02d6636bf69a80ff2159d07dfc071e7f95670d2c Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 8 Feb 2023 21:40:15 +0200 Subject: [PATCH] chore: make tests pass --- .github/workflows/addons.yaml | 19 ++--- .github/workflows/main.yaml | 71 +++++++++++++------ Tiltfile | 3 +- deployments/dev/tilt.yml | 8 ++- .../authoring-api/extractionRules.spec.js | 2 +- .../test-data/subject_extraction_rules.rego | 1 + .../spec/tweek-api/context.test.js | 2 +- e2e/integration/utils/minio.js | 6 +- services/gateway/.dockerignore | 1 - services/gateway/debug.Dockerfile | 16 ----- skaffold.yaml | 2 +- 11 files changed, 74 insertions(+), 57 deletions(-) delete mode 100644 services/gateway/debug.Dockerfile diff --git a/.github/workflows/addons.yaml b/.github/workflows/addons.yaml index 728eb78c6..7e033d6f6 100644 --- a/.github/workflows/addons.yaml +++ b/.github/workflows/addons.yaml @@ -3,17 +3,17 @@ name: Addons Tests on: push: paths: - - 'addons/**' - - '.github/workflows/addons.yaml' + - 'addons/**' + - '.github/workflows/addons.yaml' jobs: run-context-tests: runs-on: ubuntu-latest container: mcr.microsoft.com/dotnet/sdk:5.0 env: - DOTNET_CLI_TELEMETRY_OPTOUT: "true" - REDIS_TEST_CONNECTION: "redis:6379" - MONGODB_TEST_CONNECTION: "mongodb://mongo:27017" - COUCHBASE_TEST_URL: "http://couchbase:8091" + DOTNET_CLI_TELEMETRY_OPTOUT: 'true' + REDIS_TEST_CONNECTION: 'redis:6379' + MONGODB_TEST_CONNECTION: 'mongodb://mongo:27017' + COUCHBASE_TEST_URL: 'http://couchbase:8091' services: redis: @@ -23,17 +23,20 @@ jobs: mongo: image: mongo:3.6-jessie steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Test redis driver working-directory: addons/Context/Tweek.Drivers.Context.Redis.IntegrationTests run: | - dotnet test + dotnet test + - name: Test Couchbase driver working-directory: addons/Context/Tweek.Drivers.Context.Couchbase.IntegrationTests run: | curl --retry 3 --retry-max-time 10 -v http://couchbase:8091/settings/web -d port=8091 -d username=Administrator -d password=password curl --retry 3 --retry-max-time 10 -v -u Administrator:password -X POST http://couchbase:8091/pools/default/buckets -d authType=sasl -d name=testbucket -d ramQuotaMB=100 -d saslPassword=password dotnet test + - name: Test Mongo driver working-directory: addons/Context/Tweek.Drivers.Context.MongoDb.IntegrationTests run: | diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index bbb0b43b9..5c2adaf3e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -3,22 +3,24 @@ name: Main on: push: paths-ignore: - - 'docs/**' - - README.md + - 'docs/**' + - README.md jobs: build-images: runs-on: ubuntu-latest strategy: matrix: - image: ["authoring", "gateway", "publishing", "api", "editor"] + image: ['authoring', 'gateway', 'publishing', 'api', 'editor'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Login Docker DH if: ${{ github.repository == 'soluto/tweek' }} env: DOCKERHUB_TOKEN: ${{ secrets.DOCKER_TOKEN }} DOCKERHUB_USER: ${{ secrets.DOCKER_USER }} run: echo $DOCKERHUB_TOKEN | docker login --username $DOCKERHUB_USER --password-stdin + - name: build run: | echo $GITHUB_REPOSITORY @@ -26,80 +28,97 @@ jobs: docker save -o ${{ matrix.image }}-${{ github.sha }}.tar soluto/tweek-${{ matrix.image }} && \ echo cp -a ${{ matrix.image }}-${{ github.sha }}.tar .github/workflows/ cp -a ${{ matrix.image }}-${{ github.sha }}.tar .github/workflows/ + - name: 'Upload Artifact' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: build-images path: | .github/workflows/${{ matrix.image }}-${{ github.sha }}.tar run-e2e-ui: - needs: ["build-images"] + needs: ['build-images'] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Download build images - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: build-images + - name: load images into docker run: echo api publishing editor gateway authoring | xargs -n 1 | xargs -I {} docker load -i {}-$GITHUB_SHA.tar + - name: run tests run: yarn && yarn test:docker:ui + run-e2e-integration: - needs: ["build-images"] + needs: ['build-images'] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Download build images - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: build-images + - name: load images into docker run: echo api publishing editor gateway authoring | xargs -n 1 | xargs -I {} docker load -i {}-$GITHUB_SHA.tar + - name: run integration tests working-directory: e2e/integration run: | yarn && yarn start gateway && yarn test:docker run-net-tests: - needs: ["build-images"] + needs: ['build-images'] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Download build images - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: build-images + - name: load images into docker run: echo api publishing editor gateway authoring | xargs -n 1 | xargs -I {} docker load -i {}-$GITHUB_SHA.tar + - name: run smoke tests run: yarn && yarn test:docker:smoke + push-images: - needs: ["run-net-tests", "run-e2e-integration", "run-e2e-ui"] + needs: ['run-net-tests', 'run-e2e-integration', 'run-e2e-ui'] runs-on: ubuntu-latest if: ${{ github.repository == 'soluto/tweek' && !startsWith(github.ref, 'refs/heads/dependabot') }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Login Docker DH if: ${{ github.repository == 'soluto/tweek' }} env: DOCKERHUB_TOKEN: ${{ secrets.DOCKER_TOKEN }} DOCKERHUB_USER: ${{ secrets.DOCKER_USER }} run: echo $DOCKERHUB_TOKEN | docker login --username $DOCKERHUB_USER --password-stdin + - name: Download build images - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: build-images + - name: load images into docker run: echo api publishing editor gateway authoring | xargs -n 1 | xargs -I {} docker load -i {}-$GITHUB_SHA.tar + - name: push branch if: ${{ github.ref != 'refs/heads/master' }} run: | export GITHUB_BRANCH=$(echo $GITHUB_REF | sed -e "s/refs\/heads\///" -e "s/\//-/") echo api publishing editor gateway authoring | xargs -n 1 | xargs -I {} docker tag soluto/tweek-{} soluto/tweek-{}:$GITHUB_BRANCH echo api publishing editor gateway authoring | xargs -n 1 | xargs -I {} docker push soluto/tweek-{}:$GITHUB_BRANCH + - name: push master if: ${{ github.ref == 'refs/heads/master' }} run: | @@ -107,42 +126,48 @@ jobs: echo api publishing editor gateway authoring | xargs -n 1 | xargs -I {} docker push soluto/tweek-{}:latest echo api publishing editor gateway authoring | xargs -n 1 | xargs -I {} docker tag soluto/tweek-{} soluto/tweek-{}:$GITHUB_SHA echo api publishing editor gateway authoring | xargs -n 1 | xargs -I {} docker push soluto/tweek-{}:$GITHUB_SHA + - name: check versions if: ${{ github.ref == 'refs/heads/master' }} - working-directory: ".github/workflows" + working-directory: '.github/workflows' run: | ./versions_to_deploy.sh + - name: Push new api version if: ${{ github.ref == 'refs/heads/master' && env.TWEEK_API_VERSION != '' }} - working-directory: ".github/workflows" + working-directory: '.github/workflows' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | ./push-tag.sh api $TWEEK_API_VERSION + - name: Push new editor version if: ${{ github.ref == 'refs/heads/master' && env.TWEEK_EDITOR_VERSION != '' }} - working-directory: ".github/workflows" + working-directory: '.github/workflows' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | ./push-tag.sh editor $TWEEK_EDITOR_VERSION + - name: Push new authoring version if: ${{ github.ref == 'refs/heads/master' && env.TWEEK_AUTHORING_VERSION != '' }} - working-directory: ".github/workflows" + working-directory: '.github/workflows' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | ./push-tag.sh authoring $TWEEK_AUTHORING_VERSION + - name: Push new publishing version if: ${{ github.ref == 'refs/heads/master' && env.TWEEK_PUBLISHING_VERSION != '' }} - working-directory: ".github/workflows" + working-directory: '.github/workflows' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | ./push-tag.sh publishing $TWEEK_PUBLISHING_VERSION + - name: Push new gateway version if: ${{ github.ref == 'refs/heads/master' && env.TWEEK_GATEWAY_VERSION != '' }} - working-directory: ".github/workflows" + working-directory: '.github/workflows' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/Tiltfile b/Tiltfile index ec82a1b95..97ed1940e 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,5 +1,5 @@ docker_compose("./deployments/dev/tilt.yml" ) -docker_build("soluto/tweek-gateway", "services/gateway",dockerfile="services/gateway/debug.Dockerfile") +docker_build("soluto/tweek-gateway", "services/gateway",dockerfile="services/gateway/Dockerfile") docker_build("soluto/tweek-authoring", "services/authoring") docker_build("soluto/tweek-api", ".", dockerfile="TweekApi.Dockerfile") @@ -10,6 +10,7 @@ docker_build("soluto/tweek-editor", "services/editor",dockerfile="services/edito sync('services/editor/src', '/app/src'), ] ) +docker_build("soluto/tweek-bare-repo", "services/git-service/BareRepository") dc_resource('minio') dc_resource('redis') diff --git a/deployments/dev/tilt.yml b/deployments/dev/tilt.yml index 7ac3f103a..44cdea941 100644 --- a/deployments/dev/tilt.yml +++ b/deployments/dev/tilt.yml @@ -49,6 +49,7 @@ services: - publishing environment: NODE_TLS_REJECT_UNAUTHORIZED: '0' + NODE_OPTIONS: "--max-old-space-size=8192" GIT_PRIVATE_KEY_PATH: /run/secrets/tweek_ssh_private_key GIT_PUBLIC_KEY_PATH: /run/secrets/tweek_ssh_public_key GIT_URL: ssh://git@publishing/tweek/repo @@ -83,6 +84,7 @@ services: - nats - oidc-server-mock environment: + CONFIGOR_ENV: development TWEEKGATEWAY_CONFIGFILEPATH: /config/gateway.json TWEEKGATEWAY_SECURITY_AUTH_BASICAUTH_REDIRECTURLS: '[''http://localhost:8081'', ''http://localhost:3000'']' @@ -118,11 +120,13 @@ services: secrets: - source: tweek_ssh_public_key minio: - command: server /data - image: minio/minio + command: server /data --console-address ":9001" + image: minio/minio:RELEASE.2022-05-19T18-20-59Z.hotfix.30a60091e ports: - published: 4007 target: 9000 + - published: 4012 + target: 9001 secrets: - source: minio_access_key target: access_key diff --git a/e2e/integration/spec/authoring-api/extractionRules.spec.js b/e2e/integration/spec/authoring-api/extractionRules.spec.js index c543b45a5..614a56d9d 100644 --- a/e2e/integration/spec/authoring-api/extractionRules.spec.js +++ b/e2e/integration/spec/authoring-api/extractionRules.spec.js @@ -12,7 +12,7 @@ describe('authoring api extraction rules', () => { const originalRules = buf.toString(); const newRules = originalRules + '\n'; // only adding new line in order not to break the original rules - + await pollUntil( () => getObjectContentFromMinio('security/subject_extraction_rules.rego'), (res) => expect(res).to.contain(originalRules), diff --git a/e2e/integration/spec/authoring-api/test-data/subject_extraction_rules.rego b/e2e/integration/spec/authoring-api/test-data/subject_extraction_rules.rego index 004418013..063748eb6 100644 --- a/e2e/integration/spec/authoring-api/test-data/subject_extraction_rules.rego +++ b/e2e/integration/spec/authoring-api/test-data/subject_extraction_rules.rego @@ -3,6 +3,7 @@ package rules default subject = { "user": null, "group": null } subject = { "user": "admin-app", "group": "externalapps"} { + startswith(input.iss, "https://localhost:") input.aud = "tweek-openid-mock-client" input.sub = "user" } else = { "user": input.sub, "group": "default" } { diff --git a/e2e/integration/spec/tweek-api/context.test.js b/e2e/integration/spec/tweek-api/context.test.js index bcbb669ac..9688fa366 100644 --- a/e2e/integration/spec/tweek-api/context.test.js +++ b/e2e/integration/spec/tweek-api/context.test.js @@ -39,7 +39,7 @@ describe('tweek api - context', () => { }); }); - it('should return bad response when trying to append context with bad property type', async () => { + it.only('should return bad response when trying to append context with bad property type', async () => { await client .post(`/api/v2/context/${identityType}/user-1`) .send({ AgentVersion: 'not a version' }) diff --git a/e2e/integration/utils/minio.js b/e2e/integration/utils/minio.js index c7c7c39b3..6eb9a0bb1 100644 --- a/e2e/integration/utils/minio.js +++ b/e2e/integration/utils/minio.js @@ -20,10 +20,10 @@ module.exports.getObjectContentFromMinio = async (objectName) => { if (!stream) { return reject('stream is null.'); } - const contentChuncks = []; - stream.on('data', (chunk) => contentChuncks.push(chunk)); + const contentChunks = []; + stream.on('data', (chunk) => contentChunks.push(chunk)); stream.on('end', () => { - return resolve(contentChuncks.join('')); + return resolve(contentChunks.join('')); }); stream.on('error', (err) => { return reject(err); diff --git a/services/gateway/.dockerignore b/services/gateway/.dockerignore index 26c7c1107..de6e622bd 100644 --- a/services/gateway/.dockerignore +++ b/services/gateway/.dockerignore @@ -1,5 +1,4 @@ Dockerfile* -debug.Dockerfile docker-compose* .dockerignore .git diff --git a/services/gateway/debug.Dockerfile b/services/gateway/debug.Dockerfile deleted file mode 100644 index 7b081c993..000000000 --- a/services/gateway/debug.Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# syntax = docker/dockerfile:1.2 -FROM golang:1.16.3-stretch as build -WORKDIR /app - -ADD go.mod /app/go.mod -ADD go.sum /app/go.sum - -RUN --mount=id=tweek-gateway-pkgcache,type=cache,target=/go/pkg/mod go mod download -x -ADD . /app - -RUN --mount=id=tweek-gateway-build-cache,type=cache,target=/root/.cache/go-build --mount=id=tweek-gateway-pkgcache,type=cache,target=/go/pkg/mod go build -o entry \ - && go test -cover -v ./... - -RUN --mount=id=tweek-gateway-build-cache,type=cache,target=/root/.cache/go-build go build -o hcheck "tweek-gateway/healthcheck" -HEALTHCHECK --interval=5s --timeout=2s --retries=10 CMD ["/app/healthcheck"] -ENTRYPOINT [ "/app/entry" ] \ No newline at end of file diff --git a/skaffold.yaml b/skaffold.yaml index 313b2f040..e06b6bb7d 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -32,7 +32,7 @@ build: - src: '**/*.go' dest: /app docker: - dockerfile: debug.Dockerfile + dockerfile: Dockerfile - image: soluto/tweek-editor context: services/editor sync: