fix: add buffer max size and chunk size (#25)

* add buffer max size and chunk size to config files

* update testing workflows to be more reusable by moving logic to python

Author: arthur-observe

.github/workflows/AWS-EC2-Tests.yaml

@@ -26,9 +26,9 @@ on:
         options:
         - false
         - true
-  push:
-    branches:    
-      - 'test/**'
+  # push:
+  #   branches:    
+  #     - 'test/**'
   pull_request:
 
   schedule:
@@ -51,12 +51,11 @@ jobs:
       pull-requests: write
       issues: read
       checks: write
-    env: # Or as an environment variable
-      TF_VAR_OBSERVE_CUSTOMER_ID: ${{ secrets.TF_VAR_OBSERVE_CUSTOMER_ID }}
-      TF_VAR_OBSERVE_DATASTREAM_TOKEN: ${{ secrets.TF_VAR_OBSERVE_DATASTREAM_TOKEN }}
+    env:
       TF_VAR_PUBLIC_KEY: ${{ secrets.TF_VAR_PUBLIC_KEY }}
       TF_VAR_REGION: "us-west-2"
       TF_VAR_USE_BRANCH_NAME: main
+
       ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} 
       ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
       ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
@@ -68,86 +67,54 @@ jobs:
       FAIL_FIRST_TEST: false
       FAIL_SECOND_TEST: false
       WORK_DIR: test_code
-
-
+      CLOUD: aws
+      MODULE: aws_machines
       # TF_LOG: DEBUG
+
     steps:
-      - name: Value of inputs
-        run: |
-          echo "install_script_branch=${{ github.event.inputs.install_script_branch }}"
-          echo "this_repo_branch=${{ github.event.inputs.this_repo_branch }}"
-          echo "terraform_run_destroy=${{ github.event.inputs.terraform_run_destroy }}"
-          echo "fail_first_test=${{ github.event.inputs.fail_first_test }}"
-          echo "fail_second_test=${{ github.event.inputs.fail_second_test }}"
+      # AWS Login - orig role - has to occur before checkout
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.THUNDERDOME_AWS_ROLE }}
+          aws-region: ${{ env.TF_VAR_REGION }}
 
-      - name: Set variables based on inputs
-        if: github.event.inputs.terraform_run_destroy != ''
+      - name: Set code repo
         run: |
-
-          echo "TF_VAR_USE_BRANCH_NAME=${{ github.event.inputs.install_script_branch }}" >> $GITHUB_ENV
-
-          echo "THIS_REPO_BRANCH=refs/heads/${{ github.event.inputs.this_repo_branch }}" >> $GITHUB_ENV
-
-          echo "TERRAFORM_RUN_DESTROY=${{ github.event.inputs.terraform_run_destroy }}" >> $GITHUB_ENV
-        
-          echo "FAIL_FIRST_TEST=${{ github.event.inputs.fail_first_test }}" >> $GITHUB_ENV
+          if ${{ github.event.inputs.this_repo_branch != '' }}; then
+            echo "THIS_REPO_BRANCH=refs/heads/${{ github.event.inputs.this_repo_branch }}" >> $GITHUB_ENV
+          else
+            echo "THIS_REPO_BRANCH=refs/heads/${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
           
-          echo "FAIL_SECOND_TEST=${{ github.event.inputs.fail_second_test }}" >> $GITHUB_ENV
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.THIS_REPO_BRANCH }}
 
-      - name: Set Custom Environment Variables
+      - name: Set contexts
         run: |
-          # if pull request don't destroy resources
-          if ${{ github.event_name == 'pull_request' }}; then
-            echo "TERRAFORM_RUN_DESTROY=false" >> $GITHUB_ENV
-            echo "THIS_REPO_BRANCH=${{ github.ref }}" >> $GITHUB_ENV
-          fi
-
-          # value for resource names
-          echo "TF_VAR_WORKFLOW_MATRIX_VALUE=${{ matrix.test_groups }}" >> $GITHUB_ENV
-
-          # This variable tells code it running on CI server
-          echo "TF_VAR_CI=$CI" >> $GITHUB_ENV
-
-          # This variable gets just the branch name without url stuff
-          echo "GIT_HUB_SHORT_BRANCH_NAME=${GITHUB_BRANCH##*/}" >> $GITHUB_ENV
-
-          # Create directory for keys and set permissions
-          mkdir "$HOME/.ssh"
-          chmod 700 "$HOME/.ssh"
-          # variable for private key which is required for CI server to login to machines
-          echo "TF_VAR_PRIVATE_KEY_PATH=$HOME/.ssh/github_actions" >> $GITHUB_ENV
-        env:
-          GITHUB_BRANCH: ${{ github.ref }}
+          mkdir context 
+          echo '${{ toJSON(github) }}' > context/github_context.json   
+          echo '${{ toJSON(matrix) }}' > context/matrix_context.json 
+          echo '${{ secrets.PRIVATE_KEY }}' > context/private_key 
+   
+        working-directory: "${{ env.WORK_DIR }}/python_scripts"
 
-      - name: Set SSH Key
+      - name: workflow helper
         run: |
-          # variable for path to private key
-          echo "${{ secrets.PRIVATE_KEY }}" > "$TF_VAR_PRIVATE_KEY_PATH"
-          # set permissions for key file
-          chmod 600 "$TF_VAR_PRIVATE_KEY_PATH"
-          # display for troubleshooting
-          cat "$TF_VAR_PRIVATE_KEY_PATH"
+          python3 -c "from workflow_tasks import set_custom_vars; set_custom_vars(context_dir='context')"
 
-      - name: Print Environment Variables - troubleshooting
-        run: |
-          env | sort -f
+          python3 -c "from workflow_tasks import tf_override_file; tf_override_file(cloud=\"${{ env.CLOUD }}\", test_group=\"${{ matrix.test_groups }}\")"
 
-      - name: Check out repository code
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ env.THIS_REPO_BRANCH }}
+          # !!! vvvvv THIS OVERWRITES MAIN.TF FILE for specific cloud module vvvvv !!! 
+          python3 -c "from workflow_tasks import tf_main_file; tf_main_file(module=\"${{ env.MODULE }}\")"
 
-      - name: Setup Terraform
-        uses: hashicorp/setup-terraform@v2
-        with:
-          terraform_wrapper: false
+          python3 -c "from workflow_tasks import tf_output_file; tf_output_file(module=\"${{ env.MODULE }}\")"
 
-      # AWS Login - orig role
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          role-to-assume: ${{ secrets.THUNDERDOME_AWS_ROLE }}
-          aws-region: ${{ env.TF_VAR_REGION }}
+          python3 -c "from workflow_tasks import config_ini; config_ini(custid=\"${{ secrets.STAGE_CUSTOMER_ID }}\", domain=\"${{ secrets.STAGE_DOMAIN }}\", token=\"${{ secrets.STAGE_DATASTREAM_TOKEN }}\")"
+
+        working-directory: "${{ env.WORK_DIR }}/python_scripts"
 
       # GCP Login
       # This is key generated in GCP console for service account
@@ -161,33 +128,21 @@ jobs:
         with:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
 
-      - name: 'Test gcloud CLI'
-        run: 'gcloud info'
-
-      - name: terraform and config files
+      - name: Print Environment Variables - troubleshooting
         run: |
-          python3 -c "from workflow_tasks import tf_override_file; tf_override_file(cloud=\"aws\", test_group=\"${{ matrix.test_groups }}\")"
-
-          python3 -c "from workflow_tasks import tf_main_file; tf_main_file(module=\"aws_machines\")"
-
-          python3 -c "from workflow_tasks import tf_output_file; tf_output_file(module=\"aws_machines\")"
-
-          python3 -c "from workflow_tasks import config_ini; config_ini(custid=\"${{ secrets.STAGE_CUSTOMER_ID }}\", domain=\"${{ secrets.STAGE_DOMAIN }}\", token=\"${{ secrets.STAGE_DATASTREAM_TOKEN }}\")"
+          env | sort -f
 
-        working-directory: "${{ env.WORK_DIR }}/python_scripts"
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_wrapper: false
 
-      - name: terraform version
+      - name: terraform tasks
         run: |
           terraform version
-        working-directory: "${{ env.WORK_DIR }}"
 
-      - name: terraform init
-        run: |
           terraform init
-        working-directory: "${{ env.WORK_DIR }}"
 
-      - name: terraform validate
-        run: |
           terraform validate
         working-directory: "${{ env.WORK_DIR }}"
 
@@ -197,7 +152,7 @@ jobs:
         working-directory: "${{ env.WORK_DIR }}"
 
       # Run tests
-      - name: python script
+      - name: run fabric tests python script
         run: |
           # create output directory for archive files
           mkdir file_outputs
@@ -207,34 +162,27 @@ jobs:
           pip3 install -r requirements.txt
 
           # run tests 
-          fab test -a ${{ env.FAIL_FIRST_TEST }} -b ${{ env.THIS_REPO_BRANCH }}
+          fab test -a ${{ env.FAIL_FIRST_TEST }} -b ${{ env.THIS_REPO_BRANCH }} -o "1: run fabric tests python script"
         working-directory: "${{ env.WORK_DIR }}/python_scripts"
 
       - name: Retry tests
         if: ${{ env.TEST_RESULT == 'FAIL' }} 
         run: |
           # run tests
-          fab test -a ${{ env.FAIL_SECOND_TEST }} -o 2 -b ${{ env.THIS_REPO_BRANCH }}
+          fab test -a ${{ env.FAIL_SECOND_TEST }} -o "2: Retry tests"  -b ${{ env.THIS_REPO_BRANCH }}
           
         working-directory: "${{ env.WORK_DIR }}/python_scripts" 
 
-      - name: delete ini
+      - name: cleanup
         if: always()
         run: |
-          rm config.ini
-        working-directory: "${{ env.WORK_DIR }}/python_scripts"
+          rm -f python_scripts/config.ini
 
-      - name: clean_output
-        if: always()
-        run: |
-          sed -i 's/${{ secrets.STAGE_DATASTREAM_TOKEN }}/******/g' ./*
-        working-directory: "${{ env.WORK_DIR }}/python_scripts/file_outputs"
+          sed -i 's/${{ secrets.STAGE_DATASTREAM_TOKEN }}/******/g' ./python_scripts/file_outputs/*
 
-      - name: clean_logs
-        if: always()
-        run: |
-          sed -i 's/${{ secrets.STAGE_DATASTREAM_TOKEN }}/******/g' ./*
-        working-directory: "${{ env.WORK_DIR }}/python_scripts/log_outputs"
+          sed -i 's/${{ secrets.STAGE_DATASTREAM_TOKEN }}/******/g' ./python_scripts/log_outputs/*
+
+        working-directory: "${{ env.WORK_DIR }}"
 
       - name: Archive test results
         uses: actions/upload-artifact@v3