Run CI in AWS EC2 instance

.github/workflows/get-charm-paths.sh

@@ -1,12 +1,12 @@
 #!/bin/bash -x
 
-# Finds the charms in this repo, outputing them as JSON
+# Finds the charms in this repo, outputting them as JSON
 # Will return one of:
 # * the relative paths of the directories listed in `./charms`, if that directory exists
 # * "./", if the root directory has a "metadata.yaml" file
 # * otherwise, error
 #
-# Modifed from: https://stackoverflow.com/questions/63517732/github-actions-build-matrix-for-lambda-functions/63736071#63736071
+# Modified from: https://stackoverflow.com/questions/63517732/github-actions-build-matrix-for-lambda-functions/63736071#63736071
 CHARMS_DIR="./charms"
 if [ -d "$CHARMS_DIR" ];
 then

.github/workflows/integrate.yaml

@@ -4,92 +4,121 @@ name: CI
 on:
   workflow_call:
     secrets:
-      charmcraft-credentials:
+      CHARMCRAFT_CREDENTIALS:
+        required: true
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_DEFAULT_REGION:
+        required: true
+      AWS_REGION:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      GH_PERSONAL_ACCESS_TOKEN:
         required: true
 
 jobs:
-
-  lib-check:
-    name: Check libraries
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        charm:
-        - mlflow-server
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Check libs
-      uses: canonical/charming-actions/check-libraries@2.1.1
-      with:
-        charm-path: ./charms/${{ matrix.charm }}
-        credentials: "${{ secrets.charmcraft-credentials }}"
-        github-token: "${{ secrets.GITHUB_TOKEN }}"
-
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1.7.0
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2.3.3
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ${{ vars.AWS_EC2_IMAGE_ID }}
+          ec2-instance-type: ${{ vars.AWS_EC2_INSTANCE_TYPE }}
+          subnet-id: ${{ vars.AWS_DEFAULT_SUBNET_ID }}
+          security-group-id: ${{ vars.AWS_SECURITY_GROUP_ID }}
   lint:
-    name: Lint
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        charm:
-        - server
+    name: Run lint
+    needs: start-runner
+    runs-on: ${{ needs.start-runner.outputs.label }}
     steps:
-    - uses: actions/checkout@v2
-    - run: python3 -m pip install tox
-    - run: tox -e ${{ matrix.charm }}-lint
-
-  unit:
-    name: Unit tests
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        charm:
-        - server
+    - name: Check out repo
+      uses: actions/checkout@v2
+    - name: Install dependencies
+      run: sudo apt-get install -y python3-pip tox
+    - name: Lint code
+      run: tox -e lint
+  unit-tests:
+    name: Run unit tests
+    needs: [start-runner, lint]  
+    runs-on: ${{ needs.start-runner.outputs.label }}
     steps:
-    - uses: actions/checkout@v2
-    - run: python3 -m pip install tox
-    - run: tox -e ${{ matrix.charm }}-unit
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Install dependencies
+      run: sudo apt-get install -y python3-pip tox
 
-  integration:
-    name: Integration tests (microk8s)
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        charm:
-        - server
+    - name: Run unit tests
+      run: tox -e unit
+  integration-tests:
+    name: Run integration tests
+    needs: [start-runner, lint, unit-tests]
+    runs-on: ${{ needs.start-runner.outputs.label }} 
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup operator environment
-      # TODO: Unpin this when https://github.com/charmed-kubernetes/actions-operator/pull/46 is merged
-      uses: claudiubelu/actions-operator@18ebf92ae3043bd3dd15238e5d9b662d7ba08daf
-      with:
-        provider: microk8s
-        channel: 1.24/stable
-        # Pinned until this bug is resolved: https://bugs.launchpad.net/juju/+bug/1992833
-        bootstrap-options: "--agent-version=2.9.34"
-        microk8s-addons: "dns storage rbac metallb:10.64.140.43-10.64.140.49"
-    - name: Install firefox
-      run: sudo apt install -y firefox
+    - name: Check out repo
+      uses: actions/checkout@v2
 
+    - name: Setup operator environment
+      # The runner will run the code as root. There is problem when using juju under root so here is small workaround to exec as ubuntu user
+      run: |
+        chmod 777 .
+        /bin/su -c "$(pwd)/.github/workflows/setup_environment.sh" - ubuntu
+    - name: Setup microk8s
+      run: /bin/su -c "$(pwd)/.github/workflows/setup_microk8s.sh" - ubuntu
     - name: Run integration tests
-      run: sg microk8s -c "tox -e ${{ matrix.charm }}-integration"
+      run: /bin/su -c "cd $(pwd) && tox -vve integration -- --model kubeflow" - ubuntu 
 
-      # Collect debug logs if failed
-    - name: Dump Juju/k8s logs on failure
-      uses: canonical/charm-logdump-action@main
+    # On failure, capture debugging resources
+    - name: Get all
+      run: /bin/su -c "kubectl get all -A" - ubuntu 
       if: failure()
-      with:
-        app: ${{ matrix.charm }}
-        model: testing
 
-    - name: Upload HAR logs
-      uses: actions/upload-artifact@v2
-      with:
-        name: selenium-har
-        path: /tmp/selenium-*.har
+    - name: Get juju status
+      run: /bin/su -c "juju status" - ubuntu 
       if: failure()
+
+    - name: Get workload logs
+      run: /bin/su -c "kubectl logs --tail 100 -ntesting -lapp.kubernetes.io/name=mlflow-server-manager" - ubuntu 
+      if: failure()
+
+    - name: Get operator logs
+      run: /bin/su -c "kubectl logs --tail 100 -ntesting -loperator.juju.is/name=mlflow-server-manager" - ubuntu
+      if: failure()
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner 
+      - lint
+      - unit-tests
+      - integration-tests
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1.7.0
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2.3.3
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}

.github/workflows/on_pull_request.yaml

@@ -13,11 +13,21 @@ jobs:
     name: Run Tests
     uses: ./.github/workflows/integrate.yaml
     secrets:
-      charmcraft-credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }}
+      CHARMCRAFT_CREDENTIALS: ${{ secrets.CHARMCRAFT_CREDENTIALS }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
 
-  # publish runs in parallel with tests, as we always publish in this situation
+  # publish runs in parallel with tests, as we always publish in this situation 
   publish-charm:
     name: Publish Charm
     uses: ./.github/workflows/publish.yaml
     secrets:
       CHARMCRAFT_CREDENTIALS: ${{ secrets.CHARMCRAFT_CREDENTIALS }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

.github/workflows/on_push.yaml

@@ -15,7 +15,6 @@ on:
     - track/**
 
 jobs:
-
   tests:
     name: Run Tests
     uses: ./.github/workflows/integrate.yaml

.github/workflows/publish.yaml

@@ -12,6 +12,16 @@ on:
     secrets:
       CHARMCRAFT_CREDENTIALS:
         required: true
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_DEFAULT_REGION:
+        required: true
+      AWS_REGION:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      GH_PERSONAL_ACCESS_TOKEN:
+        required: true
   workflow_dispatch:
     inputs:
       destination_channel:
@@ -26,37 +36,61 @@ on:
         type: string
 
 jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ${{ vars.AWS_EC2_IMAGE_ID }}
+          ec2-instance-type: ${{ vars.AWS_EC2_INSTANCE_TYPE }}
+          subnet-id: ${{ vars.AWS_DEFAULT_SUBNET_ID }}
+          security-group-id: ${{ vars.AWS_SECURITY_GROUP_ID }}
   get-charm-paths:
     name: Generate the Charm Matrix
-    runs-on: ubuntu-20.04
+    needs: start-runner
+    runs-on: ${{ needs.start-runner.outputs.label }}
     outputs:
       charm_paths_list: ${{ steps.get-charm-paths.outputs.CHARM_PATHS_LIST }}
     steps:
       - uses: actions/checkout@v2
         with: 
           fetch-depth: 0
           ref: ${{ inputs.source_branch }}
+      - name: Install jq
+        run: sudo snap install jq
       - name: Get paths for all charms in repo
         id: get-charm-paths
         run: bash .github/workflows/get-charm-paths.sh
-
-
   publish-charm:
     name: Publish Charm
-    runs-on: ubuntu-20.04
-    needs: get-charm-paths
+    needs: [start-runner, get-charm-paths]
+    runs-on: ${{ needs.start-runner.outputs.label }}
     strategy:
       fail-fast: false
       matrix:
         charm-path: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths_list) }}
-
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
           ref: ${{ inputs.source_branch }}
-
+      - name: Setup environment
+        run: apt install -y python3 python-dev python3-dev build-essential libssl-dev libffi-dev libxml2-dev libxslt1-dev zlib1g-dev python3-pip
       - name: Select charmhub channel
         uses: canonical/charming-actions/channel@2.1.1
         id: select-channel
@@ -91,3 +125,25 @@ jobs:
           charm-path: ${{ matrix.charm-path }}
           channel: ${{ steps.parse-inputs.outputs.destination_channel }}
           tag-prefix: ${{ steps.parse-inputs.outputs.tag_prefix }}
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner 
+      - get-charm-paths
+      - publish-charm
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}

.github/workflows/release.yaml

@@ -10,9 +10,6 @@ on:
       origin-channel:
         description: 'Origin Channel'
         required: true
-      charm-name:
-        description: 'Charm subdirectory name'
-        required: true
 
 jobs:
   promote-charm:
@@ -27,5 +24,3 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           destination-channel: ${{ github.event.inputs.destination-channel }}
           origin-channel: ${{ github.event.inputs.origin-channel }}
-          tag-prefix: ${{ github.event.inputs.charm-name }}
-          charm-path: charms/${{ github.event.inputs.charm-name}}