Feature/e2e-tests add e2e tests from sidechains-tests repository

.github/workflows/devnet.yml

@@ -0,0 +1,154 @@
+name: Devnet Partner Chains Tests
+
+on:
+  schedule:
+    - cron:  '30 4 * * *'
+  workflow_dispatch:
+    inputs:
+      keyword:
+        description: 'Run tests by keyword (-k)'
+        type: string
+      latest_mc_epoch:
+        description: 'Parametrize committee tests to verify whole last MC epoch'
+        type: boolean
+      log_level:
+        description: 'Log level'
+        required: true
+        default: 'info'
+        type: choice
+        options:
+        - critical
+        - error
+        - warning
+        - info
+        - debug
+      plan:
+          description: 'XRay Test Plan'
+          type: string
+          required: false
+      execution:
+          description: 'XRay Test Execution'
+          type: string
+          required: false
+
+jobs:
+  run:
+    permissions:
+      id-token: write
+      contents: read
+
+    runs-on: [self-hosted, eks]
+
+    if: github.event.pull_request.draft == false
+
+    env:
+      TEST_ENVIRONMENT: devnet
+
+    steps:
+    - name: set ssh-agent to binary host
+      uses: webfactory/ssh-agent@v0.8.0
+      with:
+        ssh-private-key: ${{ secrets.SSH_KEY_BINARY_HOST }}
+
+    - name: Acquire AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        role-to-assume: ${{ secrets.AWS_ROLE_ARN_ }}
+        aws-region: "eu-central-1"
+
+    - name: checkout code
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event_name == 'workflow_dispatch' && github.ref_name || 'master' }}
+
+    - name: Set deployment_version as env variable
+      run: echo "DEPLOYMENT_VERSION=$(jq -r .deployment_version ./config/substrate/devnet_nodes.json)" >> $GITHUB_ENV
+
+    - name: Set XRay variables
+      run: |
+        if [ ! -z "${{ inputs.plan }}" ]; then
+          echo "TEST_PLAN=${{ inputs.plan }}" >> $GITHUB_ENV
+        elif [ ! -z "${{ inputs.execution }}" ]; then
+          echo "TEST_EXECUTION=${{ inputs.execution }}" >> $GITHUB_ENV
+        else
+          echo "TEST_PLAN=ETCM-7235" >> $GITHUB_ENV
+        fi
+
+    - name: set report_to_xray env variable
+      run: |
+        echo "REPORT_TO_XRAY=$([[ '${{ github.event_name }}' == 'schedule' || -n '${{ github.event.inputs.plan }}' || -n '${{ github.event.inputs.execution }}' ]] && echo true || echo false )" >> $GITHUB_ENV
+
+    - name: Setup Earthly
+      uses: ./.github/earthly-setup
+      with:
+        ssh_key: ${{ secrets.SUBSTRATE_REPO_SSH_KEY }}
+        config_tar: ${{ secrets.EARTHLY_TAR }}
+
+    - name: run
+      env:
+        EARTHLY_BUILD_ARGS: "CI_RUN=true"
+        FORCE_COLOR: 1
+        SLACK_REF_NAME: ${{ github.event_name == 'schedule' && format('{0}/v{1}', env.TEST_ENVIRONMENT, env.DEPLOYMENT_VERSION) || github.ref_name }}
+        LOG_LEVEL: ${{ inputs.log_level }}
+        KEYWORD: ${{ inputs.keyword }}
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+        LATEST_MC_EPOCH: ${{ (inputs.latest_mc_epoch == true) && true || false }}
+        REPORT_TO_SLACK: ${{ github.ref == 'refs/heads/master' && 'true' || 'false' }}
+        GITHUB_ACTOR_USERNAME: ${{ github.event_name == 'schedule' && 'nightly' || github.actor }}
+        XRAY_CLIENT_ID: ${{ secrets.XRAY_CLIENT_ID }}
+        XRAY_CLIENT_SECRET: ${{ secrets.XRAY_CLIENT_SECRET }}
+        XRAY_API_BASE_URL: ${{ secrets.XRAY_API_BASE_URL }}
+        JIRA_URL: ${{ secrets.JIRA_URL }}
+        MARKERS: "not active_flow and not passive_flow"
+      run: |
+        earthly \
+          --secret AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \
+          --secret AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
+          --secret AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
+          --secret SLACK_WEBHOOK_URL="$SLACK_WEBHOOK_URL" \
+          --secret XRAY_CLIENT_ID="$XRAY_CLIENT_ID" \
+          --secret XRAY_CLIENT_SECRET="$XRAY_CLIENT_SECRET" \
+          --secret XRAY_API_BASE_URL="$XRAY_API_BASE_URL" \
+          --secret JIRA_URL="$JIRA_URL" \
+          --ssh-auth-sock="$SSH_AUTH_SOCK" \
+          ./E2E-tests+report \
+          --log_level="${LOG_LEVEL:-"info"}" \
+          --env=${{ env.TEST_ENVIRONMENT }} \
+          --stack=${{ env.TEST_ENVIRONMENT }} \
+          --keyword="${KEYWORD:-"test_"}" \
+          --repository ${{ github.repository }} \
+          --job_url="$JOB_URL" \
+          --slack_ref_name $SLACK_REF_NAME \
+          --latest_mc_epoch=$LATEST_MC_EPOCH \
+          --report_to_slack=$REPORT_TO_SLACK \
+          --github_actor_username="$GITHUB_ACTOR_USERNAME" \
+          --plan="$TEST_PLAN" \
+          --execution="$TEST_EXECUTION" \
+          --report_to_xray=$REPORT_TO_XRAY \
+          --decrypt=true \
+          --markers="$MARKERS"
+
+    - name: Archive Debug Log
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: debug_log
+        retention-days: 15
+        overwrite: true
+        path: debug.log
+
+    - name: Setup Node
+      uses: actions/setup-node@v4
+      if: ${{ !cancelled() }}
+      with:
+        node-version: 'latest'
+
+    - name: Install Ctrf
+      if: ${{ !cancelled() }}
+      run: npm install github-actions-ctrf
+
+    - name: Generate Summary Report
+      if: ${{ !cancelled() }}
+      run: |
+        npx github-actions-ctrf ctrf-report.json

.github/workflows/staging.yml

@@ -0,0 +1,159 @@
+name: Staging Partner Chains Tests
+
+on:
+  schedule:
+    - cron: '0 4 * * *'
+  push:
+    branches:
+      - feature/e2e-tests
+  workflow_dispatch:
+    inputs:
+      keyword:
+        description: 'Run tests by keyword (-k)'
+        type: string
+      latest_mc_epoch:
+        description: 'Parametrize committee tests to verify whole last MC epoch'
+        type: boolean
+      log_level:
+        description: 'Log level'
+        required: true
+        default: 'info'
+        type: choice
+        options:
+        - critical
+        - error
+        - warning
+        - info
+        - debug
+      plan:
+          description: 'XRay Test Plan'
+          type: string
+          required: false
+      execution:
+          description: 'XRay Test Execution'
+          type: string
+          required: false
+
+jobs:
+  run:
+    permissions:
+      id-token: write
+      contents: read
+
+
+    runs-on: [self-hosted, eks]
+
+    if: github.event.pull_request.draft == false
+
+    env:
+      TEST_ENVIRONMENT: staging
+
+    steps:
+    - name: set ssh-agent to binary host
+      uses: webfactory/ssh-agent@v0.8.0
+      with:
+        ssh-private-key: ${{ secrets.SSH_KEY_BINARY_HOST }}
+
+    - name: Acquire AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        role-to-assume: ${{ secrets.AWS_ROLE_ARN_ }}
+        aws-region: "eu-central-1"
+
+    - name: checkout code
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event_name == 'workflow_dispatch' && github.ref_name || 'feature/e2e-tests' }}
+
+    - name: Set deployment_version as env variable
+      run: echo "DEPLOYMENT_VERSION=$(jq -r .deployment_version ./config/substrate/staging_nodes.json)" >> $GITHUB_ENV
+
+    - name: Set XRay variables
+      run: |
+        if [ ! -z "${{ inputs.plan }}" ]; then
+          echo "TEST_PLAN=${{ inputs.plan }}" >> $GITHUB_ENV
+        elif [ ! -z "${{ inputs.execution }}" ]; then
+          echo "TEST_EXECUTION=${{ inputs.execution }}" >> $GITHUB_ENV
+        else
+          echo "TEST_PLAN=ETCM-8452" >> $GITHUB_ENV
+        fi
+
+    - name: set report_to_xray env variable
+      run: |
+        echo "REPORT_TO_XRAY=$([[ '${{ github.event_name }}' == 'schedule' || -n '${{ github.event.inputs.plan }}' || -n '${{ github.event.inputs.execution }}' ]] && echo true || echo false )" >> $GITHUB_ENV
+
+    - name: Setup Earthly
+      uses: ./.github/earthly-setup
+      with:
+        ssh_key: ${{ secrets.SUBSTRATE_REPO_SSH_KEY }}
+        config_tar: ${{ secrets.EARTHLY_TAR }}
+
+    - name: run
+      env:
+        EARTHLY_BUILD_ARGS: "CI_RUN=true"
+        FORCE_COLOR: 1
+        SLACK_REF_NAME: ${{ github.event_name == 'schedule' && format('{0}/v{1}', env.TEST_ENVIRONMENT, env.DEPLOYMENT_VERSION) || github.ref_name }}
+        LOG_LEVEL: ${{ inputs.log_level }}
+        KEYWORD: ${{ inputs.keyword }}
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+        LATEST_MC_EPOCH: ${{ (github.event.schedule || inputs.latest_mc_epoch == true) && true || false }}
+        REPORT_TO_SLACK: ${{ github.ref == 'refs/heads/master' && 'true' || 'false' }}
+        GITHUB_ACTOR_USERNAME: ${{ github.event_name == 'schedule' && 'nightly' || github.actor }}
+        XRAY_CLIENT_ID: ${{ secrets.XRAY_CLIENT_ID }}
+        XRAY_CLIENT_SECRET: ${{ secrets.XRAY_CLIENT_SECRET }}
+        XRAY_API_BASE_URL: ${{ secrets.XRAY_API_BASE_URL }}
+        JIRA_URL: ${{ secrets.JIRA_URL }}
+        MARKERS: "not active_flow and not passive_flow"
+      run: |
+        cd E2E-tests
+        earthly \
+          --secret AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \
+          --secret AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
+          --secret AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
+          --secret SLACK_WEBHOOK_URL="$SLACK_WEBHOOK_URL" \
+          --secret XRAY_CLIENT_ID="$XRAY_CLIENT_ID" \
+          --secret XRAY_CLIENT_SECRET="$XRAY_CLIENT_SECRET" \
+          --secret XRAY_API_BASE_URL="$XRAY_API_BASE_URL" \
+          --secret JIRA_URL="$JIRA_URL" \
+          --ssh-auth-sock="$SSH_AUTH_SOCK" \
+          +report \
+          --log_level="${LOG_LEVEL:-"info"}" \
+          --env=${{ env.TEST_ENVIRONMENT }} \
+          --stack=${{ env.TEST_ENVIRONMENT }} \
+          --keyword="${KEYWORD:-"test_"}" \
+          --repository ${{ github.repository }} \
+          --job_url="$JOB_URL" \
+          --slack_ref_name $SLACK_REF_NAME \
+          --latest_mc_epoch=$LATEST_MC_EPOCH \
+          --report_to_slack=$REPORT_TO_SLACK \
+          --github_actor_username="$GITHUB_ACTOR_USERNAME" \
+          --plan="$TEST_PLAN" \
+          --execution="$TEST_EXECUTION" \
+          --report_to_xray=$REPORT_TO_XRAY \
+          --decrypt=true \
+          --markers="$MARKERS"
+
+    - name: Archive Debug Log
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: debug_log
+        retention-days: 15
+        overwrite: true
+        path: debug.log
+
+    - name: Setup Node
+      uses: actions/setup-node@v4
+      if: ${{ !cancelled() }}
+      with:
+        node-version: 'latest'
+
+    - name: Install Ctrf
+      if: ${{ !cancelled() }}
+      run: npm install github-actions-ctrf
+
+    - name: Generate Summary Report
+      if: ${{ !cancelled() }}
+      run: |
+        npx github-actions-ctrf ctrf-report.json

E2E-tests/.gitignore

@@ -0,0 +1,10 @@
+.obsidian
+**/__pycache__/
+.decrypted~*
+*.decrypted
+venv/
+contractlog.json
+reports
+debug.log
+.vscode/settings.json
+my_env/

E2E-tests/.sops.yaml

@@ -0,0 +1,9 @@
+creation_rules:
+    - path_regex: secrets\/[^/]+\/[^l].*|secrets\/[^/]+\/l[^o].*|secrets\/[^/]+\/lo[^c].*|secrets\/[^/]+\/loc[^a].*|secrets\/[^/]+\/loca[^l].*|secrets\/[^/]+\/local[^/].*
+      kms: >-
+        arn:aws:kms:eu-central-1:689191102645:key/ca20bdac-ce35-4859-a2ef-e796df8d0abc
+      pgp: >-
+        15AC8EA84FC2A5AE768FFD753CEBBA453DE5BCFD,
+        33E4D64843E72C6C8A665703199BE5B70A2AACE3,
+        C5C51001A3BF0F4031C37EC99681FA65B010848C,
+        3A10EB97F2C401D93CC4685D6B0DDB72B8DEA45D