Release/0.6.0

.flake8

@@ -1,3 +1,4 @@
 [flake8]
 #ignore = ...
 max-line-length=180
+per-file-ignores = __init__.py:F401

.github/workflows/build-pipeline.yml

@@ -10,6 +10,8 @@ on:
   workflow_dispatch:
 
 env:
+  POETRY_VERSION: "1.3.1"
+  PYTHON_VERSION: "3.10"
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
@@ -21,18 +23,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: Install Poetry
-        uses: abatilo/actions-poetry@v2.0.0
+        uses: abatilo/actions-poetry@v2
         with:
-          poetry-version: 1.1.8
+          poetry-version: ${{ env.POETRY_VERSION }}
       - name: Get version
         id: get-version
         run: |
-          echo "::set-output name=current_version::$(poetry version | awk '{print $2}')"
+          echo "current_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT
           echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV
       - name: Bump pre-alpha version
         # If triggered by push to a feature branch
@@ -79,9 +81,6 @@ jobs:
         run: |
           poetry run pylint podaac
           poetry run flake8 podaac
-      - name: Install dependencies
-        run: |
-          poetry add --dev papermill
       - name: Test and coverage
         run: |
           poetry run pytest --junitxml=build/reports/pytest.xml --cov=podaac/ --cov-report=xml:build/reports/coverage.xml -m "not aws and not integration" tests/
@@ -99,9 +98,20 @@ jobs:
             -Dsonar.tests=tests/
             -Dsonar.projectName=podaac-concise
             -Dsonar.projectVersion=${{ env.software_version }}
-            -Dsonar.python.version=3.9
+            -Dsonar.python.version=${{ env.PYTHON_VERSION }}
+      - name: Run Snyk as a blocking step
+        uses: snyk/actions/python-3.10@master
+        env:
+            SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          command: test
+          args: >
+              --org=${{ secrets.SNYK_ORG_ID }}
+              --project-name=${{ github.repository }}
+              --severity-threshold=high
+              --fail-on=all
       - name: Run Snyk on Python
-        uses: snyk/actions/python-3.8@master
+        uses: snyk/actions/python-3.10@master
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
@@ -131,7 +141,7 @@ jobs:
           git tag -a "${{ env.software_version }}" -m "Version ${{ env.software_version }}"
           git push origin "${{ env.software_version }}"
       - name: Publish UMM-S with new version
-        uses: podaac/cmr-umm-updater@0.2.1
+        uses: podaac/cmr-umm-updater@0.2.3
         if: |
           github.ref == 'refs/heads/main'    ||
           startsWith(github.ref, 'refs/heads/release')
@@ -141,22 +151,23 @@ jobs:
           env: ${{ env.venue }}
           version: ${{ env.software_version }}
           timeout: 60
+          disable_removal: 'true'
         env:
           cmr_user: ${{secrets.CMR_USER}}
           cmr_pass: ${{secrets.CMR_PASS}}
       - name: Build Docs
         run: |
           poetry run sphinx-build -b html ./docs docs/_build/
       - name: Publish Docs
-        uses: JamesIves/github-pages-deploy-action@4.1.5
+        uses: JamesIves/github-pages-deploy-action@v4
         with:
           branch: gh-pages # The branch the action should deploy to.
           folder: docs/_build/ # The folder the action should deploy.
           target-folder: ${{ env.software_version }}
       - name: Build Python Artifact
         run: |
           poetry build
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: python-artifact
           path: dist/*
@@ -177,15 +188,15 @@ jobs:
           poetry publish
       - name: Log in to the Container registry
         if: ${{ !startsWith(github.ref, 'refs/heads/feature') }}
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Extract metadata (tags, labels) for Docker
         if: ${{ !startsWith(github.ref, 'refs/heads/feature') }}
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
@@ -198,7 +209,8 @@ jobs:
           ${GITHUB_WORKSPACE}/.github/workflows/wait-for-pypi.py ${{env.pyproject_name}}[harmony]==${{ env.software_version }}
       - name: Build and push Docker image
         if: ${{ !startsWith(github.ref, 'refs/heads/feature') }}
-        uses: docker/build-push-action@v2
+        id: docker-push
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: docker/Dockerfile
@@ -208,22 +220,30 @@ jobs:
           pull: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-      - name: Run Snyk on Docker Image
-        if: ${{ !startsWith(github.ref, 'refs/heads/feature') }}
-        # Snyk can be used to break the build when it detects vulnerabilities.
-        # In this case we want to upload the issues to GitHub Code Scanning
-        continue-on-error: true
-        uses: snyk/actions/docker@master
-        env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.software_version }}
-          args: >
-            --severity-threshold=high
-            --file=./docker/Dockerfile
-            --sarif-file-output=docker.sarif
-      - name: Upload result to GitHub Code Scanning
-        if: ${{ !startsWith(github.ref, 'refs/heads/feature') }}
-        uses: github/codeql-action/upload-sarif@v1
-        with:
-          sarif_file: ./
+# As of 2023/01/23 these steps below for scanning the Docker image with Snyk are failing. I've tried both the official Snyk
+# action https://github.com/snyk/actions/tree/master/docker and this method below of manually calling the CLI.
+# The error when using the official Snyk action is
+#   "Could not detect package manager for file: ./docker/Dockerfile"
+# The error when trying to run it manually is
+#   "The image does not exist for the current platform"
+# Commenting these steps out for now so they don't fail the build. Will try revisiting again in the future when Snyk is updated.
+# 
+#       - uses: snyk/actions/setup@master
+#         if: |
+#           steps.docker-push.conclusion == 'success'
+#       - name: Run Snyk on Docker Image
+#         if: |
+#           steps.docker-push.conclusion == 'success'
+#         # Snyk can be used to break the build when it detects vulnerabilities.
+#         # In this case we want to upload the issues to GitHub Code Scanning
+#         continue-on-error: true
+#         run: |
+#           snyk test --severity-threshold=high --file=./docker/Dockerfile --sarif-file-output=docker.sarif --docker ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.software_version }}
+#         env:
+#           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+#       - name: Upload result to GitHub Code Scanning
+#         if: |
+#           steps.docker-push.conclusion == 'success'
+#         uses: github/codeql-action/upload-sarif@v2
+#         with:
+#           sarif_file: ./

.github/workflows/jupyter_test.yml

@@ -1,4 +1,4 @@
-# This is the main build pipeline that verifies and publishes the software
+# This is a test pipeline, that verifies and updates the associations in cmr
 name: Jupyter Test
 # Controls when the workflow will run
 on:
@@ -17,16 +17,142 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: 3.9
+      - name: Set environment variables
+        run: |
+          echo "UAT_OUTPUT_FILE=jupyter_notebooks/output/uat_output_json.txt" >> $GITHUB_ENV
+          echo "OPS_OUTPUT_FILE=jupyter_notebooks/output/ops_output_json.txt" >> $GITHUB_ENV
+          echo "OUTPUT_DIR=jupyter_notebooks/output" >> $GITHUB_ENV
+          echo "UAT_SUCCESS_FILE=uat_success.txt" >> $GITHUB_ENV
+          echo "OPS_SUCCESS_FILE=ops_success.txt" >> $GITHUB_ENV
+          echo "UAT_FAILURE_FILE=uat_fail.txt" >> $GITHUB_ENV
+          echo "OPS_FAILURE_FILE=ops_fail.txt" >> $GITHUB_ENV
+          echo "SLACK_REPORT_CONTENT=" >> $GITHUB_ENV
+      - name: Set environment variables for notebook
+        run: |
+          echo "OPS_USERNAME=${{ secrets.EDL_USER }}" >> $GITHUB_ENV
+          echo "OPS_PASSWORD=${{ secrets.EDL_PASS }}" >> $GITHUB_ENV
+          echo "UAT_USERNAME=${{ secrets.EDL_USER }}" >> $GITHUB_ENV
+          echo "UAT_PASSWORD=${{ secrets.EDL_PASS }}" >> $GITHUB_ENV
       - name: Create prerequisites
         run: |
-          mkdir current_project
-          touch current_project/test_in.txt
-          echo "ASCATC-L2-Coastal" > current_project/test_in.txt
-          mkdir tests/jupyter/notebooks/output
+          mkdir ${{ env.OUTPUT_DIR }}
+          realpath ${{ env.OUTPUT_DIR }}
       - name: Install dependencies
         run: |
-          pip install papermill
-      - name: Run Jupyter notebook
+          pip3 install --upgrade pip
+          pip3 install papermill
+          pip3 install xarray
+          pip3 install jupyter
+          pip3 install black
+          pip3 install matplotlib
+          pip3 install netCDF4
+          pip3 install git+https://github.com/nasa/harmony-py.git
+          pip3 install git+https://github.com/podaac/cmr-umm-updater.git
+          pip3 install git+https://github.com/podaac/cmr-association-diff.git@6193079a14e36f4c9526aa426015c2b6be41f0e2
+          pip3 install python-dateutil --upgrade
+      - name: Run CMR Association diff scripts
+        run: |
+          cmr_association_diff -e uat -t service -a "cmr/uat_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o ${{ env.UAT_OUTPUT_FILE }} --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }}
+          cmr_association_diff -e ops -t service -a "cmr/ops_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o ${{ env.OPS_OUTPUT_FILE }} --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }}
+      - name: Run Jupyter notebooks
+        run: |
+          python3 "notebook_test.py" -n "jupyter_notebooks/harmony_concise_api_test.ipynb" -e uat -i ${{ env.UAT_OUTPUT_FILE }} -o ${{ env.OUTPUT_DIR }}
+          python3 "notebook_test.py" -n "jupyter_notebooks/harmony_concise_api_test.ipynb" -e ops -i ${{ env.OPS_OUTPUT_FILE }} -o ${{ env.OUTPUT_DIR }}
+      - name: Check UAT files
+        id: check_UAT_output_files
+        run: |
+          echo "::set-output name=SUCCESS_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "::set-output name=FAIL_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "::set-output name=SUCCESS_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo "::set-output name=FAIL_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo
+          echo
+          echo "Success File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "Failed File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "Success Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo
+          echo "Failed Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+      - name: Check OPS files
+        id: check_OPS_output_files
+        run: |
+          echo "::set-output name=SUCCESS_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "::set-output name=FAIL_EXISTS::$(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "::set-output name=SUCCESS_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo "::set-output name=FAIL_CONTENT::$(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo
+          echo
+          echo "Success File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "Failed File: $(if [ -f ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} ]; then echo "true"; else echo "false"; fi)"
+          echo "Success Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+          echo
+          echo "Failed Content: $(cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }} | sed 's/$/,/' | tr '\n' ' ')"
+      - name: Slack Report update - Add info for UAT check - SUCCESS
+        if: |
+          steps.check_UAT_output_files.outputs.SUCCESS_EXISTS == 'true'
+        run: |
+          echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\nJupyter-test UAT result: SUCCESS\nSuccessfully ran Concise Notebook on the following collections:\n${{ steps.check_UAT_output_files.outputs.SUCCESS_CONTENT }}\n" >> $GITHUB_ENV
+      - name: Slack Report update - Add info for UAT check - FAILURE
+        if: |
+          steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true'
+        run: |
+          echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\nJupyter-test UAT result: FAILURE\nFailed to run Concise Notebook on the following collections:\n${{ steps.check_UAT_output_files.outputs.FAIL_CONTENT }}\n" >> $GITHUB_ENV
+      - name: Slack Report update - Add info for OPS check - SUCCESS
+        if: |
+          steps.check_OPS_output_files.outputs.SUCCESS_EXISTS == 'true'
+        run: |
+          echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\nJupyter-test OPS result: SUCCESS\nSuccessfully ran Concise Notebook on the following collections:\n${{ steps.check_OPS_output_files.outputs.SUCCESS_CONTENT }}\n" >> $GITHUB_ENV
+      - name: Slack Report update - Add info for OPS check - FAILURE
+        if: |
+          steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true'
+        run: |
+          echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\nJupyter-test OPS result: FAILURE\nFailed to run Concise Notebook on the following collections:\n${{ steps.check_OPS_output_files.outputs.FAIL_CONTENT }}\n" >> $GITHUB_ENV
+      - name: Slack Report update - Add action link on Failure
+        if: |
+          steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true' ||
+          steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true'
+        run: |
+          echo "SLACK_REPORT_CONTENT=${{ env.SLACK_REPORT_CONTENT }}\nExecution link:\ngithub.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV
+          echo ${{ env.SLACK_REPORT_CONTENT }}
+      - name: Send Report to Slack
+        uses: slackapi/slack-github-action@v1.21.0
+        if: |
+          env.SLACK_REPORT_CONTENT != ''
+        with:
+          payload: |
+            {
+              "status": "${{ env.SLACK_REPORT_CONTENT }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BOT_WEBHOOK }}
+      - name: Remove UAT collections on FAIL
+        if: |
+          steps.check_UAT_output_files.outputs.FAIL_EXISTS == 'true'
+        run: |
+          python3 "remove_association.py" -p POCLOUD -n 'PODAAC Concise' -e uat -i '${{ env.OUTPUT_DIR }}/${{ env.UAT_FAILURE_FILE }}' --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }}
+      - name: Remove OPS collections on FAIL
+        if: |
+          steps.check_OPS_output_files.outputs.FAIL_EXISTS == 'true'
+        run: |
+          python3 "remove_association.py" -p POCLOUD -n 'PODAAC Concise' -e ops -i '${{ env.OUTPUT_DIR }}/${{ env.OPS_FAILURE_FILE }}' --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }}
+      - name: Update UAT collections on SUCCESS
+        if: |
+          steps.check_UAT_output_files.outputs.SUCCESS_EXISTS == 'true'
+        # Add a new line to the EOF if there isn't any, so the merging will start in a new line not after the last character
+        run: |
+          vi -escwq ./cmr/uat_associations.txt
+          cat ${{ env.OUTPUT_DIR }}/${{ env.UAT_SUCCESS_FILE }} >> ./cmr/uat_associations.txt
+          git config user.name "${GITHUB_ACTOR}"
+          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git commit -am "Update uat_associations.txt with new collections"
+          git push
+      - name: Update OPS collections on SUCCESS
+        if: |
+          steps.check_OPS_output_files.outputs.SUCCESS_EXISTS == 'true'
+        # Add a new line to the EOF if there isn't any, so the merging will start in a new line not after the last character
         run: |
-          python3 "./tests/jupyter/notebook_test.py" -n "./tests/jupyter/notebooks/harmony_concise_api_test.ipynb" -e uat -i ./current_project/test_in.txt -o ./tests/jupyter/notebooks//output
-
+          vi -escwq ./cmr/ops_associations.txt
+          cat ${{ env.OUTPUT_DIR }}/${{ env.OPS_SUCCESS_FILE }} >> ./cmr/ops_associations.txt
+          git config user.name "${GITHUB_ACTOR}"
+          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git commit -am "Update ops_associations.txt with new collections"
+          git push