diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..90b48ec73 --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +max-line-length=88 +count=True +ignore=E203,W503,E712 +extend-exclude=_version.py,lib,*_docs,geoips_dev_utils +docstring-convention=numpy +rst-roles=class,func,ref +rst-directives=envvar,exception +rst-substitutions=version +statistics=True +per-file-ignores = + /*/interfaces/__init__.py:F401 diff --git a/.github/versions/tagged_version b/.github/versions/tagged_version new file mode 100644 index 000000000..63e799cf4 --- /dev/null +++ b/.github/versions/tagged_version @@ -0,0 +1 @@ +1.14.1 diff --git a/.github/versions/upcoming_version b/.github/versions/upcoming_version new file mode 100644 index 000000000..4ea8ad87e --- /dev/null +++ b/.github/versions/upcoming_version @@ -0,0 +1 @@ +1.14.3 diff --git a/.github/workflows/brassy-notes.yaml b/.github/workflows/brassy-notes.yaml new file mode 100644 index 000000000..d91e01c33 --- /dev/null +++ b/.github/workflows/brassy-notes.yaml @@ -0,0 +1,25 @@ +name: Brassy + +on: + # Triggers the workflow when pull request closed (either closed + # as non-planned, or actually approved and merged.) + pull_request: + types: + - closed + branch: main + # Allows run of this workflow manually from the Actions tab + # Must be merged to default before it will be available to manually run. + workflow_dispatch: + +jobs: + brassy-notes: + name: Brassy + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-brassy-notes.yaml@main + # Only run this if the pull request was merged, not just closed. + if: github.event.pull_request.merged == true + permissions: + contents: write + pull-requests: write + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy-docs.yaml b/.github/workflows/deploy-docs.yaml new file mode 100644 index 000000000..b32188262 --- /dev/null +++ b/.github/workflows/deploy-docs.yaml @@ -0,0 +1,36 @@ +name: Deploy docs + +# 1. Brassy - on merge of v*-version-release +# * generate release notes +# * commit and push to v*-add-rst-release-note +# * open PR to default branch +# 2. Tag and Release - on merge of v*-add-rst-release-note +# * Tag current version +# * Release just tagged version +# 3. Package and Publish - on published release (from #2) +# * Build wheel +# * Publish to pypi +# 4. Deploy docs - on published release (from #2) +# * pip install geoips +# * pip install plugin repo +# * build docs with geoips/docs/build_docs.sh +# * deploy docs with geoips/docs/deploy_pages.sh + +on: + # Triggers the workflow on published release + release: + types: + - published + # Allows run of this workflow manually from the Actions tab + # Must be merged to default before it will be available to manually run. + workflow_dispatch: + +jobs: + deploy-docs: + name: Deploy docs + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-deploy-docs.yaml@main + permissions: + contents: write + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy-ghpages-docs.yaml b/.github/workflows/deploy-ghpages-docs.yaml deleted file mode 100644 index 266ab8fde..000000000 --- a/.github/workflows/deploy-ghpages-docs.yaml +++ /dev/null @@ -1,133 +0,0 @@ -name: Build and deploy docs - -defaults: - run: - shell: bash - -on: - # Allows run of this workflow manually from the Actions tab - # Must be merged to dev before it will be available to manually run. - workflow_dispatch: - # Run on each published release - release: - types: [published] - -jobs: - deploy_ghpages: - runs-on: ${{ vars.RUNNER }} - steps: - - name: Setup Python - uses: actions/setup-python@v4 # v4is the latest for setup-python - with: - python-version: "pypy-3.10" # I believe 3.10 is the latest for actions - check-latest: true - - name: Update pip - run: pip install --upgrade pip - - name: Checkout geoips repo default branch - uses: actions/checkout@v3 # v3 is the latest for checkout - with: - fetch-depth: 0 - repository: ${{ vars.ORGANIZATION }}/geoips - ref: ${{ vars.DEFAULT_BRANCH }} - token: ${{ secrets.GEOIPS_TOKEN }} - path: geoips_default_branch - - name: pip install geoips_default_branch[doc] - run: | - which pip - pip uninstall -y geoips - pip install $PWD/geoips_default_branch[doc] - ret=$? - echo "${ret}" - python -c "import geoips; print(geoips.__version__)" - - name: Checkout current repo current branch - uses: actions/checkout@v3 # v3 is the latest for checkout - with: - fetch-depth: 0 - path: geoips_plugin_repo - - name: pip install geoips plugin repo - run: | - which pip - curr_repo=`echo "$GITHUB_REPOSITORY" | sed "s,$GITHUB_REPOSITORY_OWNER/,,"` - # Do not re-install if this is the geoips repo - if [[ "$curr_repo" != "geoips" ]]; then - pip uninstall -y $curr_repo - pip install -v $PWD/geoips_plugin_repo - ret=$? - echo "${ret}" - python -c "import $curr_repo; print($curr_repo.__version__)" - fi - - name: create_plugin_registries - run: | - echo "create plugin registries" - which create_plugin_registries - create_plugin_registries - ret=$? - echo "${ret}" - - name: Run build docs - run: | - which pip - echo "Build html and pdf docs" - # Ie, get 'recenter_tc' out of 'NRLMMD-GEOIPS/recenter_tc' - curr_repo=`echo "$GITHUB_REPOSITORY" | sed "s,$GITHUB_REPOSITORY_OWNER/,,"` - # Call build_docs.sh from the geoips default branch - # Pass in - # the path to the current plugin repo, - # the name of the current repo, - # "html_pdf" to indicate producing both html and pdf output, and - # the path to the docs template directory in the geoips default branch. - ./geoips_default_branch/docs/build_docs.sh ./geoips_plugin_repo $curr_repo html_pdf ./geoips_default_branch/docs - ret=$? - if [[ "${ret##*:}" != *"0"* ]]; then - echo "::error::Building html and pdf docs ${ret##*:}" - exit 1 - fi - - name: Run deploy docs - run: | - which pip - echo "Deploy docs to github pages" - # GITHUB_REPOSITORY, GITHUB_REPOSITORY_OWNER, and GITHUB_SERVER_URL are all - # standard github variables defined for every github actions run. - # Ie, get 'recenter_tc' out of 'NRLMMD-GEOIPS/recenter_tc' - curr_repo=`echo "$GITHUB_REPOSITORY" | sed "s,$GITHUB_REPOSITORY_OWNER/,,"` - # ie, https://github.com/NRLMMD-GEOIPS - GEOIPS_REPO_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY_OWNER" - # Remove ghpages_repo dir if it exists. - # deploy_pages.sh will not run if ghpages_repo exists. - if [[ -d ./ghpages_repo ]]; then - echo "::group::remove_ghpages_before" - echo "Remove ghpages_repo before deploy" - echo "rm -rfv ./ghpages_repo" - rm -rfv ./ghpages_repo - echo "::endgroup::" - rmret=$? - if [[ "${rmret##*:}" != *"0"* ]]; then - echo "::error::Removing ./ghpages_repo before deploy ${rmret##*:}" - exit 1 - fi - fi - # Use the deploy_pages.sh script from the geoips default branch. - # Pass in the - # current plugin repo path, - # the geoips repo url, and - # a path to a new directory for the ghpages checkout/push - # the name of the current repository - ./geoips_default_branch/docs/deploy_pages.sh ./geoips_plugin_repo $GEOIPS_REPO_URL ./ghpages_repo $curr_repo - ret=$? - if [[ "${ret##*:}" != *"0"* ]]; then - echo "::error::Deploying docs to GitHub pages ${ret##*:}" - exit 1 - fi - # Remove ghpages_repo dir after running deploy_pages.sh. - # deploy_pages.sh will not run if ghpages_repo exists. - if [[ -d ./ghpages_repo ]]; then - echo "::group::remove_ghpages_after" - echo "Remove ghpages_repo after deploy" - echo "rm -rfv ./ghpages_repo" - rm -rfv ./ghpages_repo - echo "::endgroup::" - rmret=$? - if [[ "${rmret##*:}" != *"0"* ]]; then - echo "::error::Removing ./ghpages_repo after deploy ${rmret##*:}" - exit 1 - fi - fi diff --git a/.github/workflows/doc-lint-test.yaml b/.github/workflows/doc-lint-test.yaml deleted file mode 100644 index 356ec8c82..000000000 --- a/.github/workflows/doc-lint-test.yaml +++ /dev/null @@ -1,645 +0,0 @@ -# Builds an image that contains GeoIPS built with `pip install .[doc,lint,test]` -name: Build Documentation, Lint, and Test - -env: - # The registry that will be used to store all images - # Override from repo or org level to use a different registry - DOCKER_REGISTRY: ghcr.io - # The path to the GeoIPS package in DOCLINTTEST images - BASE_GEOIPS_PATH: /packages/geoips - # Allow overriding the doclinttest image tag used for plugin packages - # Allows using a different version of the doc, lint, and test code - PLUGIN_DOCLINTTEST_TAG: doclinttest-stable - -defaults: - run: - shell: bash - -on: - # Triggers when code is pushed to "main" branch - # This should only happen when PRs are merged - push: - branches: - - main - - dev-* - - # Triggers the workflow when pull request created and updated - pull_request: - - # Triggers when a new release is created - release: - types: - - published - - # Allows run of this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - # Set variables that should be accessible to all jobs - set_variables: - runs-on: ${{ vars.RUNNER }} - env: - CURR_REPO: ${{ github.event.repository.name }} - outputs: - # The name of the untagged docker images to be created by this workflow - IMAGE_NAME: ${{ steps.set_image_name.outputs.IMAGE_NAME }} - # The tag to use for the doclinttest images - DOCLINTTEST_TAG: ${{ steps.set_doclinttest_tag.outputs.DOCLINTTEST_TAG }} - # The image to use for doclinttest jobs - DOCLINTTEST_IMAGE: ${{ steps.set_doclinttest_image.outputs.DOCLINTTEST_IMAGE }} - # Flags for enabling/disabling various parts of the workflow - # Default to enabled unless overridden by a repository-level variable - CI_BUILD_SPHINX_HTML: ${{ steps.get_job_flags.outputs.CI_BUILD_SPHINX_HTML }} - CHECK_NEW_RELEASE_NOTE: ${{ steps.get_job_flags.outputs.CHECK_NEW_RELEASE_NOTE }} - CI_BUILD_SPHINX_PDF: ${{ steps.get_job_flags.outputs.CI_BUILD_SPHINX_PDF }} - CI_LINT_BANDIT: ${{ steps.get_job_flags.outputs.CI_LINT_BANDIT }} - CI_LINT_DOC8: ${{ steps.get_job_flags.outputs.CI_LINT_DOC8 }} - CI_LINT_FLAKE8: ${{ steps.get_job_flags.outputs.CI_LINT_FLAKE8 }} - CI_LINT_BLACK: ${{ steps.get_job_flags.outputs.CI_LINT_BLACK }} - CI_TEST_INTERFACES: ${{ steps.get_job_flags.outputs.CI_TEST_INTERFACES }} - CI_TEST_PYTEST_SHORT: ${{ steps.get_job_flags.outputs.CI_TEST_PYTEST_SHORT }} - steps: - - name: Set the image name - id: set_image_name - run: | - # Create an image name using the organiztion and repository names, lowercase - image_name=${DOCKER_REGISTRY}/${GITHUB_REPOSITORY,,} - echo "IMAGE_NAME=${image_name}" - echo "IMAGE_NAME=${image_name}" >> ${GITHUB_ENV} - echo "IMAGE_NAME=${image_name}" >> ${GITHUB_OUTPUT} - - name: Set doclinttest image tag - id: set_doclinttest_tag - run: | - echo "CURR_REPO: ${CURR_REPO}" - # If the current repository is geoips, use the image we're about to build - # for running tests and building docs. - if [[ "${CURR_REPO}" == "geoips" ]]; then - echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" - echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" >> ${GITHUB_ENV} - echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" >> ${GITHUB_OUTPUT} - else - echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" - echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" >> ${GITHUB_ENV} - echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" >> ${GITHUB_OUTPUT} - fi - - name: Set doclinttest image - id: set_doclinttest_image - run: | - owner=${{ github.repository_owner }} - owner=${owner,,} - doclinttest_image_name=${DOCKER_REGISTRY}/${owner}/geoips:${DOCLINTTEST_TAG} - echo "DOCLINTTEST_IMAGE=${DOCKER_REGISTRY}/${owner}/geoips:${DOCLINTTEST_TAG}" - echo "DOCLINTTEST_IMAGE=${DOCKER_REGISTRY}/${owner}/geoips:${DOCLINTTEST_TAG}" >> ${GITHUB_ENV} - echo "DOCLINTTEST_IMAGE=${DOCKER_REGISTRY}/${owner}/geoips:${DOCLINTTEST_TAG}" >> ${GITHUB_OUTPUT} - - name: Get job flags - id: get_job_flags - run: | - # Define an array of flags and their corresponding secret values - flags=(CHECK_NEW_RELEASE_NOTE\ - CI_BUILD_SPHINX_HTML \ - CI_BUILD_SPHINX_PDF \ - CI_LINT_BANDIT \ - CI_LINT_DOC8 \ - CI_LINT_FLAKE8 \ - CI_LINT_BLACK \ - CI_TEST_INTERFACES \ - CI_TEST_PYTEST_SHORT) - for flag in "${flags[@]}"; do - # Dynamically construct the variable name and access the corresponding secret - case $flag in - CHECK_NEW_RELEASE_NOTE) val="${{ vars.CHECK_NEW_RELEASE_NOTE }}" ;; - CI_BUILD_SPHINX_HTML) val="${{ vars.CI_BUILD_SPHINX_HTML }}" ;; - CI_BUILD_SPHINX_PDF) val="${{ vars.CI_BUILD_SPHINX_PDF }}" ;; - CI_LINT_BANDIT) val="${{ vars.CI_LINT_BANDIT }}" ;; - CI_LINT_DOC8) val="${{ vars.CI_LINT_DOC8 }}" ;; - CI_LINT_FLAKE8) val="${{ vars.CI_LINT_FLAKE8 }}" ;; - CI_LINT_BLACK) val="${{ vars.CI_LINT_BLACK }}" ;; - CI_TEST_INTERFACES) val="${{ vars.CI_TEST_INTERFACES }}" ;; - CI_TEST_PYTEST_SHORT) val="${{ vars.CI_TEST_PYTEST_SHORT }}" ;; - esac - # Check if the secret value is "true" and set the environment variable and output - if [ "$val" == "false" ]; then - echo "${flag}=false" - echo "${flag}=false" >> $GITHUB_ENV - echo "${flag}=false" >> $GITHUB_OUTPUT - else - echo "${flag}=true" - echo "${flag}=true" >> $GITHUB_ENV - echo "${flag}=true" >> $GITHUB_OUTPUT - fi - done - - print_run_flags: - runs-on: ${{ vars.RUNNER }} - needs: [set_variables] - steps: - - name: Print run flags - run: | - for var in $(echo '${{ toJson(needs.set_variables.outputs) }}' | jq -r 'keys[]'); do - value=$(echo '${{ toJson(needs.set_variables.outputs) }}' | jq -r --arg var "$var" '.[$var]') - echo "$var=$value" - done - - # Build the base GeoIPS doclinttest image - # This image is used in all subsequent tests - # It is only built for the main GeoIPS package but is used by all repositories - # - # Within this image, the GeoIPS package is located in /packages/geoips and is - # installed using `pip install .[doc,lint,test]`. Scripts from geoips can be called - # from /packages/geoips. - build_doclinttest_image: - runs-on: ${{ vars.RUNNER }} - needs: set_variables - if: ${{ github.event.repository.name == 'geoips' }} - env: - IMAGE_NAME: ${{ needs.set_variables.outputs.IMAGE_NAME }} - BUILDCACHE_IMAGE: ${{ needs.set_variables.outputs.IMAGE_NAME }}:buildcache - DOCLINTTEST_IMAGE: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - # Can't write to registry without this - permissions: - contents: read - packages: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup BuildX - uses: docker/setup-buildx-action@v3 - - - name: Login to the Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.DOCKER_REGISTRY }} - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - - # This cache allows the second push below when on the main branch - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Create tags - run: | - tags="${DOCLINTTEST_IMAGE},${BUILDCACHE_IMAGE}" - if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then - tags="${tags},${IMAGE_NAME}:doclinttest-stable" - else - tags="${tags},${IMAGE_NAME}:doclinttest-latest" - fi - echo "PUSH_TAGS=${tags}" - echo "PUSH_TAGS=${tags}" >> $GITHUB_ENV - - - name: Build and Push image - uses: docker/build-push-action@v6 - with: - context: . - target: doclinttest - push: true - tags: ${{ env.PUSH_TAGS }} - file: "Dockerfile" - cache-from: type=registry,ref=${{ env.BUILDCACHE_IMAGE }} - cache-to: type=registry,ref=ghcr.io/${{ env.BUILDCACHE_IMAGE }},mode=max - - -################################################# -# OLD Release Note NOT Added -################################################# - check_no_edits_to_rst_release_note: - if: > - always() && - needs.set_variables.outputs.CHECK_NEW_RELEASE_NOTE == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - env: - CURR_REPO: ${{ github.event.repository.name }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Configure Git safe directory - run: git config --global --add safe.directory $PWD - - name: Detect new and modified release files - run: | - #update_this_file="$(cat update_this_release_note)" - release_notes_dir="$PWD/docs/source/releases" #/$(dirname $update_this_file)" - echo "Checking for no edits to the .rst release note in dir $release_notes_dir" - echo "git -C . diff --name-only --diff-filter=AM remotes/origin/main -- $release_notes_dir/*.rst" - ret=$(git -C . diff --name-only --diff-filter=AM remotes/origin/main -- $release_notes_dir/*.rst) - for file in ${ret[@]}; do - echo " $file" - done - if [ -z "$ret" ]; then - echo "PASSED" - else - echo "FAILED: Release note changes detected" - echo "Please use brassy to enter change logs in $release_notes_dir" - echo "DO NOT EDIT .RST FILES DIRECTLY :))" - exit 1 - fi - -################################################# -# Release Note Added -################################################# - check_new_release_note: - if: > - always() && - github.ref != 'refs/heads/main' && - needs.set_variables.outputs.CHECK_NEW_RELEASE_NOTE == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - env: - CURR_REPO: ${{ github.event.repository.name }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Configure Git safe directory - run: git config --global --add safe.directory $PWD - - name: Detect new and modified release files - run: | - echo "Checking for new and modified release note files in $PWD" - #update_this_file="$(cat update_this_release_note)" - release_notes_dir="$PWD/docs/source/releases/latest" #/$(dirname $update_this_file)" - ls $release_notes_dir - #current_release_note=`cat update_this_release_note` - echo "git -C . diff --name-only --diff-filter=AM remotes/origin/main -- $release_notes_dir/*.yaml" - ret=$(git -C . diff --name-only --diff-filter=AM remotes/origin/main -- $release_notes_dir/*.yaml) - echo "Release note files modified this PR:" - for file in ${ret[@]}; do - echo " $file" - done - if [ -z "$ret" ]; then - echo "FAILED: No yaml release note changes detected" - echo "Please use brassy to enter change logs in $release_notes_dir" - exit 1 - else - echo "PASSED" - fi - -################################################# -# Documentation builds -################################################# - build_sphinx_html: - if: > - always() && - needs.set_variables.outputs.CI_BUILD_SPHINX_HTML == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - env: - CURR_REPO: ${{ github.event.repository.name }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Configure Git safe directory - run: git config --global --add safe.directory $PWD - - - name: Install package - run: pip install . - - - name: Build release notes from yaml files - id: release-note-generator - run: | - # update_this_file="$(cat $BASE_GEOIPS_PATH/update_this_release_note)" - # release_notes_dir="$(dirname $update_this_file)/$(basename $update_this_file .rst)" - release_notes_dir="$PWD/docs/source/releases/latest" #/$(dirname $update_this_file)" - update_this_file="$PWD/docs/source/releases/latest.rst" - #release_version="$(basename $update_this_file .rst)" - release_version="latest" - echo "Running brassy on directory: $release_notes_dir" - brassy --release-version $release_version --no-rich --output-file $update_this_file $release_notes_dir - echo "Done writing!" - echo "release_note_file=$update_this_file" >> $GITHUB_OUTPUT - - name: Pinken release note - run: pink ${{ steps.release-note-generator.outputs.release_note_file }} - - - name: Upload release note - uses: actions/upload-artifact@v4 - with: - name: Release Note - path: ${{ steps.release-note-generator.outputs.release_note_file }} - if-no-files-found: error - - - name: Run build html - id: build-html-docs - run: | - echo "Build html docs" - # Call build_docs.sh from the geoips default branch - # Pass in - # the path to the current plugin repo, - # the name of the current repo, - # "html_only" to indicate producing only html output, and - # the path to the docs template directory in the geoips default branch. - $BASE_GEOIPS_PATH/docs/build_docs.sh . $CURR_REPO html_only $BASE_GEOIPS_PATH/docs - ret=$? - if [[ "${ret##*:}" != *"0"* ]]; then - echo "::error::Building html docs ${ret##*:}" - exit 1 - fi - echo "BUILT_DOCS_PATH=$PWD/build/sphinx/html" >> $GITHUB_OUTPUT - - name: Upload HTML Docs - uses: actions/upload-artifact@v4 - with: - name: HTML Docs - path: ${{ steps.build-html-docs.outputs.BUILT_DOCS_PATH }} - if-no-files-found: error - -# This job does not currently work because latex is not installed in the image. I think -# that to get this job working would only require installing texlive-base but other -# texlive packages might also be required. -# -# Additionally, this job is extremely slow. Building the PDF documentation is time -# consuming and should probably only be run on releases. Building the HTML documentation -# should be enough to catch any problems that crop up. -# -# build_sphinx_pdf: -# if: ${{ needs.set_variables.outputs.CI_DISABLE_BUILD_SPHINX_PDF == 'false' }} -# runs-on: ${{ vars.RUNNER }} -# needs: [build_doclinttest_image, set_variables] -# container: -# image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} -# credentials: -# username: ${{ secrets.DOCKER_REGISTRY_USER }} -# password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} -# env: -# CURR_REPO: ${{ github.event.repository.name }} -# steps: -# - name: Checkout code -# uses: actions/checkout@v4 -# -# - name: Configure Git safe directory -# run: git config --global --add safe.directory $PWD -# -# - name: Install geoips documentation packages -# run: pip install -e .[doc] -# -# - name: Run build pdf -# run: | -# echo "Build pdf docs" -# # Call build_docs.sh from the geoips default branch -# # Pass in -# # the path to the current plugin repo, -# # the name of the current repo, -# # "pdf_only" to indicate producing only pdf output, and -# # the path to the docs template directory in the geoips default branch. -# ./docs/build_docs.sh . $CURR_REPO pdf_only ./docs -# ret=$? -# if [[ "${ret##*:}" != *"0"* ]]; then -# echo "::error::Building pdf docs ${ret##*:}" -# exit 1 -# fi - -################################################# -# Linting -################################################# - lint_bandit: - if: > - always() && - needs.set_variables.outputs.CI_LINT_BANDIT == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Run bandit - shell: bash -l {0} - run: | - echo "::group::bandit_analysis" - echo "BANDIT analysis of code" - $BASE_GEOIPS_PATH/tests/utils/check_code.sh bandit ./ - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to bandit violations, return code ${ret}" - exit 1 - fi - - lint_doc8: - if: > - always() && - needs.set_variables.outputs.CI_LINT_DOC8 == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Run doc8 - shell: bash -l {0} - run: | - echo "::group::doc8_analysis" - if [ ! -d "./docs/source/new-docs" ]; then - echo "No new-docs directory found in ${CURR_REPO}" - exit 0 - fi - echo "Doc8 analysis of code" - doc8 --max-line-length=120 ./docs/source/new-docs/ - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to doc8 violations, return code ${ret}" - exit 1 - fi - - lint_flake8: - if: > - always() && - needs.set_variables.outputs.CI_LINT_FLAKE8 == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Run flake8 - shell: bash -l {0} - run: | - echo "::group::flake8_analysis" - echo "FLAKE8 analysis of code" - $BASE_GEOIPS_PATH/tests/utils/check_code.sh flake8 . - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to flake8 violations, return code ${ret}" - exit 1 - fi - - lint_black: - if: > - always() && - needs.set_variables.outputs.CI_LINT_BLACK == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Run code check script black - shell: bash -l {0} - run: | - echo "::group::black_analysis" - echo "BLACK analysis of code" - black --version - $BASE_GEOIPS_PATH/tests/utils/check_code.sh black ./ - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to black violations, return code ${ret}" - exit 1 - fi - -################################################# -# Code tests -################################################# - test_interfaces: - if: > - always() && - needs.set_variables.outputs.CI_TEST_INTERFACES == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Configure Git safe directory - run: git config --global --add safe.directory $PWD - - name: Pip install package - run: pip install -v . - - name: create_plugin_registries - run: create_plugin_registries - - name: Run code check script interfaces - run: | - $BASE_GEOIPS_PATH/tests/utils/check_code.sh interfaces geoips - ret=$? - echo "::group::interface_analysis" - echo "INTERFACES analysis of code" - echo "${ret}" - echo "::endgroup::" - if [[ "${ret##*:}" != *"0"* ]]; then - echo "::error::due to interface violations ${ret##*:}" - exit 1 - fi - - test_pytest_short: - if: > - always() && - needs.set_variables.outputs.CI_TEST_PYTEST_SHORT == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - env: - CURR_REPO: ${{ github.event.repository.name }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Configure Git safe directory - run: git config --global --add safe.directory $PWD - - name: Pip install package - run: pip install -v . - - name: create_plugin_registries - run: create_plugin_registries - # Run unit tests for this repo if they exist - - name: Run pytest unit tests - run: | - echo "::group::pytest_unit_test" - echo "Pytest short unit tests of ${CURR_REPO}" - echo "which pytest" - which pytest - if [ ! -d "./tests/unit_tests" ]; then - echo "No unit tests found in ${CURR_REPO}" - exit 0 - fi - echo "pytest -q ./tests/unit_tests" - pytest -q ./tests/unit_tests - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to geoips repo pytest errors, return code ${ret}" - exit 1 - fi - - name: Run base GeoIPS pytest unit tests - if: ${{ env.CURR_REPO != 'geoips' }} - run: | - echo "::group::pytest_unit_test" - echo "Pytest short unit tests of ${CURR_REPO} using core GeoIPS tests" - echo "which pytest" - which pytest - echo "pytest -q $BASE_GEOIPS_PATH/tests/unit_tests" - pytest -q $BASE_GEOIPS_PATH/tests/unit_tests - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to geoips repo pytest errors, return code ${ret}" - exit 1 - fi diff --git a/.github/workflows/doc-test.yaml b/.github/workflows/doc-test.yaml new file mode 100644 index 000000000..7ada5d81e --- /dev/null +++ b/.github/workflows/doc-test.yaml @@ -0,0 +1,477 @@ +# Builds an image that contains GeoIPS built with `pip install .[doc,lint,test]` +name: Build Documentation and Test + +# NOTE: Various bits of CI run on particularly-named branches +# v*-version-release : auto-generated final release branch, +# merge kicks off new-brassy-note +# v*-update-rst-release-note: auto-generated branch in new-brassy-note + +env: + # REQUIRED organization level variables used in this workflow. + # ${{ github.event.repository.default_branch }} is used throughout + # rather than main to generalize default branch + # (removed DEFAULT_BRANCH org var) + # DOCKER_REGISTRY: URL to the docker registry + # DOCKER_REGISTRY_USER: user to sign into DOCKER_REGISTRY + # DOCKER_REGISTRY_TOKEN: token for USER user to sign into DOCKER_REGISTRY + # RUNNER: default runner + # UPLOAD_GITHUB_ARTIFACTS: true/false, specify whether to upload artifacts to github + # USE_DOCKER_BUILDX_ACTION: true/false, specify whether to use the docker buildx action + + # OPTIONAL organization level variables: + # # Some system implementations may require different runners for building and + # # running docker containers - allow specifying separately. Defaults to vars.RUNNER + # # if these are not defined. Note env context nor strings are supported in the + # # runs-on field, so we MUST use the vars context for the default. + # RUNNER_DOCKERBUILD: runner to build docker containers + # RUNNER_DOCKERRUN: runner to run docker containers + + # The path to the GeoIPS package in DOCLINTTEST images + BASE_GEOIPS_PATH: /packages/geoips + # Allow overriding the doclinttest image tag used for plugin packages + # Allows using a different version of the doc, lint, and test code + PLUGIN_DOCLINTTEST_TAG: ${{ vars.PLUGIN_DOCLINTTEST_TAG || 'doclinttest-stable' }} + # Allow installing in editable mode for some packages that require it + # If set to the string "true" will install packages with `pip install -e .` + INSTALL_EDITABLE: ${{ vars.INSTALL_EDITABLE || 'false' }} + +defaults: + run: + shell: bash + +on: + # Triggers the workflow when pull request created and updated + pull_request: + # Allows run of this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + # Set variables that should be accessible to all jobs + set_variables: + runs-on: ${{ vars.RUNNER }} + env: + CURR_REPO: ${{ github.event.repository.name }} + outputs: + # The name of the untagged docker images to be created by this workflow + IMAGE_NAME: ${{ steps.set_image_name.outputs.IMAGE_NAME }} + # The name of the repository + REPO_NAME: ${{ steps.set_image_name.outputs.REPO_NAME }} + # The tag to use for the doclinttest images + DOCLINTTEST_TAG: ${{ steps.set_doclinttest_tag.outputs.DOCLINTTEST_TAG }} + # The image to use for doclinttest jobs + DOCLINTTEST_IMAGE: ${{ steps.set_doclinttest_image.outputs.DOCLINTTEST_IMAGE }} + # Flags for enabling/disabling various parts of the workflow + # Default to enabled unless overridden by a repository-level variable + CI_BUILD_SPHINX_HTML: ${{ steps.get_job_flags.outputs.CI_BUILD_SPHINX_HTML }} + CI_BUILD_SPHINX_PDF: ${{ steps.get_job_flags.outputs.CI_BUILD_SPHINX_PDF }} + CI_TEST_INTERFACES: ${{ steps.get_job_flags.outputs.CI_TEST_INTERFACES }} + CI_TEST_PYTEST_SHORT: ${{ steps.get_job_flags.outputs.CI_TEST_PYTEST_SHORT }} + steps: + - name: Set the image name + id: set_image_name + run: | + repo_name=$(echo ${{ github.repository }} | cut -d'/' -f2) + # Create an image name using the organiztion and repository names, lowercase + image_name=${{ vars.DOCKER_REGISTRY_USER }}/geoips + echo "REPO_NAME=${repo_name}" + echo "REPO_NAME=${repo_name}" >> ${GITHUB_ENV} + echo "REPO_NAME=${repo_name}" >> ${GITHUB_OUTPUT} + echo "IMAGE_NAME=${image_name}" + echo "IMAGE_NAME=${image_name}" >> ${GITHUB_ENV} + echo "IMAGE_NAME=${image_name}" >> ${GITHUB_OUTPUT} + - name: Set doclinttest image tag + id: set_doclinttest_tag + run: | + echo "CURR_REPO: ${CURR_REPO}" + # If the current repository is geoips, use the image we're about to build + # for running tests and building docs. + if [[ "${CURR_REPO}" == "geoips" ]]; then + echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" + echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" >> ${GITHUB_ENV} + echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" >> ${GITHUB_OUTPUT} + else + echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" + echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" >> ${GITHUB_ENV} + echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" >> ${GITHUB_OUTPUT} + fi + - name: Set doclinttest image + id: set_doclinttest_image + run: | + owner=${{ github.repository_owner }} + owner=${owner,,} + doclinttest_image_name=${{ env.IMAGE_NAME }}:${DOCLINTTEST_TAG} + echo "DOCLINTTEST_IMAGE=${{ env.IMAGE_NAME }}:${DOCLINTTEST_TAG}" + echo "DOCLINTTEST_IMAGE=${{ env.IMAGE_NAME }}:${DOCLINTTEST_TAG}" >> ${GITHUB_ENV} + echo "DOCLINTTEST_IMAGE=${{ env.IMAGE_NAME }}:${DOCLINTTEST_TAG}" >> ${GITHUB_OUTPUT} + - name: Get job flags + id: get_job_flags + run: | + # Define an array of flags and their corresponding secret values + flags=(CI_BUILD_SPHINX_HTML \ + CI_BUILD_SPHINX_PDF \ + CI_TEST_INTERFACES \ + CI_TEST_PYTEST_SHORT) + for flag in "${flags[@]}"; do + # Dynamically construct the variable name and access the corresponding secret + case $flag in + CI_BUILD_SPHINX_HTML) val="${{ vars.CI_BUILD_SPHINX_HTML }}" ;; + CI_BUILD_SPHINX_PDF) val="${{ vars.CI_BUILD_SPHINX_PDF }}" ;; + CI_TEST_INTERFACES) val="${{ vars.CI_TEST_INTERFACES }}" ;; + CI_TEST_PYTEST_SHORT) val="${{ vars.CI_TEST_PYTEST_SHORT }}" ;; + esac + # Check if the secret value is "true" and set the environment variable and output + if [ "$val" == "false" ]; then + echo "${flag}=false" + echo "${flag}=false" >> $GITHUB_ENV + echo "${flag}=false" >> $GITHUB_OUTPUT + else + echo "${flag}=true" + echo "${flag}=true" >> $GITHUB_ENV + echo "${flag}=true" >> $GITHUB_OUTPUT + fi + done + + print_run_flags: + runs-on: ${{ vars.RUNNER }} + needs: [set_variables] + steps: + - name: Print run flags + run: | + echo "github.ref_name ${{ github.ref_name }}" + echo "github.ref ${{ github.ref }}" + echo "github.base_ref ${{ github.base_ref }}" + echo "github.head_ref ${{ github.head_ref }}" + echo "vars.RUNNER ${{ vars.RUNNER }}" + echo "vars.RUNNER_DOCKERBUILD ${{ vars.RUNNER_DOCKERBUILD }}" + echo "vars.RUNNER_DOCKERRUN ${{ vars.RUNNER_DOCKERRUN }}" + echo "github.event.repository.default_branch ${{ github.event.repository.default_branch }}" + echo "github.event.repository.name ${{ github.event.repository.name }}" + echo "github.event.pull_request.merged ${{ github.event.pull_request.merged }}" + echo "github.run_id ${{ github.run_id }}" + echo "github.run_attempt ${{ github.run_attempt }}" + echo "vars.USE_CONDA_FOR_ACTIONS ${{ vars.USE_CONDA_FOR_ACTIONS }}" + echo ${{ github.context }} + + for var in $(echo '${{ toJson(needs.set_variables.outputs) }}' | jq -r 'keys[]'); do + value=$(echo '${{ toJson(needs.set_variables.outputs) }}' | jq -r --arg var "$var" '.[$var]') + echo "$var=$value" + done + echo "Target ref:" + if [[ "${{ github.base_ref }}" != "" ]]; then + target_ref=${{ github.base_ref }} + else + target_ref=${{ github.event.repository.default_branch }} + fi + echo "remotes/origin/${target_ref}" + + # Build the base GeoIPS doclinttest image + # This image is used in all subsequent tests + # It is only built for the main GeoIPS package but is used by all repositories + # + # Within this image, the GeoIPS package is located in /packages/geoips and is + # installed using `pip install .[doc,lint,test]`. Scripts from geoips can be called + # from /packages/geoips. + build_doclinttest_image: + runs-on: ${{ vars.RUNNER_DOCKERBUILD || vars.RUNNER }} + needs: set_variables + if: > + always() && + vars.USE_CONDA_FOR_ACTIONS != 'true' && + github.event.repository.name == 'geoips' + env: + IMAGE_NAME: ${{ needs.set_variables.outputs.IMAGE_NAME }} + BUILDCACHE_IMAGE: ${{ needs.set_variables.outputs.IMAGE_NAME }}:buildcache + DOCLINTTEST_IMAGE: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} + # Can't write to registry without this + permissions: + contents: read + packages: write + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup BuildX + uses: docker/setup-buildx-action@v3 + + - name: Login to the Registry + uses: docker/login-action@v3 + with: + registry: ${{ vars.DOCKER_REGISTRY }} + username: ${{ vars.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + + # This cache allows the second push below when on the default branch + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Create tags + run: | + tags="${DOCLINTTEST_IMAGE},${BUILDCACHE_IMAGE}" + echo "GITHUB REF: ${GITHUB_REF}" + echo "Default Branch: refs/heads/${{ github.event.repository.default_branch }}" + echo "runner os: ${{ runner.os }}" + echo "github sha: ${{ github.sha }}" + echo "build cache image: ${{ env.BUILDCACHE_IMAGE }}" + if [[ "${GITHUB_REF}" == "refs/heads/${{ github.event.repository.default_branch }}" ]]; then + stableorlatest_image="${IMAGE_NAME}:doclinttest-stable" + tags="${tags},${stableorlatest_image}" + else + stableorlatest_image="${IMAGE_NAME}:doclinttest-latest" + tags="${tags},${stableorlatest_image}" + fi + # PUSH_TAGS is used with the docker buildx action + echo "PUSH_TAGS=${tags}" + echo "PUSH_TAGS=${tags}" >> $GITHUB_ENV + # STABLEORLATEST_IMAGE is only used with the explicit docker build command. + # It is the full path to the stable or latest image tag. + # The explicit docker build command requires pushing each image separately + # to the registry - so we need to track the stable/latest tag in addition + # to the DOCLINTTEST_IMAGE and BUILDCACHE_IMAGE tags + echo "STABLEORLATEST_IMAGE=${stableorlatest_image}" + echo "STABLEORLATEST_IMAGE=${stableorlatest_image}" >> $GITHUB_ENV + + # If using docker buildx was requested via organization variable, + # DO NOT run this section. + - name: Build and push docker image with explicit docker build + if: ${{ vars.USE_DOCKER_BUILDX_ACTION == 'false' }} + # > makes newlines into spaces, so single command on multiple lines. + run: > + which docker; + docker build + --file Dockerfile + --tag ${DOCLINTTEST_IMAGE} + --tag ${BUILDCACHE_IMAGE} + --tag ${STABLEORLATEST_IMAGE} + --target doclinttest + .; + docker push ${BUILDCACHE_IMAGE}; + docker push ${DOCLINTTEST_IMAGE}; + docker push ${STABLEORLATEST_IMAGE}; + - name: Build and push docker image with docker buildx action + if: ${{ vars.USE_DOCKER_BUILDX_ACTION != 'false' }} + uses: docker/build-push-action@v6 + with: + context: . + target: doclinttest + push: true + tags: ${{ env.PUSH_TAGS }} + file: "Dockerfile" + cache-from: type=registry,ref=${{ env.BUILDCACHE_IMAGE }} + cache-to: type=registry,ref=${{ vars.DOCKER_REGISTRY }}/${{ env.BUILDCACHE_IMAGE }},mode=max + + ################################################# + # Documentation builds - requires pip installed geoips, so use container + ################################################# + build_sphinx_html: + if: > + always() && + vars.USE_CONDA_FOR_ACTIONS != 'true' && + needs.set_variables.outputs.CI_BUILD_SPHINX_HTML == 'true' && + (needs.build_doclinttest_image.result == 'success' || + needs.build_doclinttest_image.result == 'skipped') + runs-on: ${{ vars.RUNNER_DOCKERRUN || vars.RUNNER }} + needs: [build_doclinttest_image, set_variables] + container: + image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} + credentials: + username: ${{ vars.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + env: + CURR_REPO: ${{ github.event.repository.name }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure Git safe directory + run: git config --global --add safe.directory $PWD + + - name: Install package + run: | + if [ "${INSTALL_EDITABLE}" = "true" ]; then + pip install -e . + else + pip install . + fi + + - name: Run build html + id: build-html-docs + run: | + # If we are specifically "template_basic_plugin" repo, then we must use + # my_package as the package name. Otherwise the package name matches + # the plugin name. + run_on_package=$CURR_REPO + if [[ "$CURR_REPO" == "template_basic_plugin" ]]; then + run_on_package=my_package + fi + echo "Build html docs" + # Call build_docs.sh from the geoips default branch + # Pass in + # the path to the current plugin repo, + # the name of the current repo, + # "html_only" to indicate producing only html output, and + # the path to the docs template directory in the geoips default branch. + $BASE_GEOIPS_PATH/docs/build_docs.sh . $run_on_package html_only $BASE_GEOIPS_PATH/docs + ret=$? + if [[ "${ret##*:}" != *"0"* ]]; then + echo "::error::Building html docs ${ret##*:}" + exit 1 + fi + echo "BUILT_DOCS_PATH=$PWD/build/sphinx/html" >> $GITHUB_OUTPUT + - name: Upload HTML Docs + if: ${{ vars.UPLOAD_GITHUB_ARTIFACTS == 'true' }} + uses: actions/upload-artifact@v4 + with: + name: HTML Docs + path: ${{ steps.build-html-docs.outputs.BUILT_DOCS_PATH }} + if-no-files-found: error + + # This job does not currently work because latex is not installed in the image. I think + # that to get this job working would only require installing texlive-base but other + # texlive packages might also be required. + # + # Additionally, this job is extremely slow. Building the PDF documentation is time + # consuming and should probably only be run on releases. Building the HTML documentation + # should be enough to catch any problems that crop up. + # + # build_sphinx_pdf: + # if: ${{ needs.set_variables.outputs.CI_DISABLE_BUILD_SPHINX_PDF == 'false' }} + # runs-on: ${{ vars.RUNNER_DOCKERRUN || vars.RUNNER }} + # needs: [build_doclinttest_image, set_variables] + # container: + # image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} + # credentials: + # username: ${{ secrets.DOCKER_REGISTRY_USER }} + # password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + # env: + # CURR_REPO: ${{ github.event.repository.name }} + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + # + # - name: Configure Git safe directory + # run: git config --global --add safe.directory $PWD + # + # - name: Install geoips documentation packages + # run: pip install -e .[doc] + # + # - name: Run build pdf + # run: | + # echo "Build pdf docs" + # # Call build_docs.sh from the geoips default branch + # # Pass in + # # the path to the current plugin repo, + # # the name of the current repo, + # # "pdf_only" to indicate producing only pdf output, and + # # the path to the docs template directory in the geoips default branch. + # ./docs/build_docs.sh . $CURR_REPO pdf_only ./docs + # ret=$? + # if [[ "${ret##*:}" != *"0"* ]]; then + # echo "::error::Building pdf docs ${ret##*:}" + # exit 1 + # fi + + ################################################# + # Code tests - requires pip installed geoips, so use container + ################################################# + test_interfaces: + if: > + always() && + vars.USE_CONDA_FOR_ACTIONS != 'true' && + needs.set_variables.outputs.CI_TEST_INTERFACES == 'true' && + (needs.build_doclinttest_image.result == 'success' || + needs.build_doclinttest_image.result == 'skipped') + runs-on: ${{ vars.RUNNER_DOCKERRUN || vars.RUNNER }} + needs: [build_doclinttest_image, set_variables] + container: + image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} + credentials: + username: ${{ vars.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Configure Git safe directory + run: git config --global --add safe.directory $PWD + - name: Pip install package + run: pip install -v . + - name: create_plugin_registries + run: create_plugin_registries + - name: Run code check script interfaces + run: | + $BASE_GEOIPS_PATH/tests/utils/check_code.sh interfaces geoips + ret=$? + echo "::group::interface_analysis" + echo "INTERFACES analysis of code" + echo "${ret}" + echo "::endgroup::" + if [[ "${ret##*:}" != *"0"* ]]; then + echo "::error::due to interface violations ${ret##*:}" + exit 1 + fi + + test_pytest_short: + if: > + always() && + vars.USE_CONDA_FOR_ACTIONS != 'true' && + needs.set_variables.outputs.CI_TEST_PYTEST_SHORT == 'true' && + (needs.build_doclinttest_image.result == 'success' || + needs.build_doclinttest_image.result == 'skipped') + runs-on: ${{ vars.RUNNER_DOCKERRUN || vars.RUNNER }} + needs: [build_doclinttest_image, set_variables] + container: + image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} + credentials: + username: ${{ vars.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + env: + CURR_REPO: ${{ github.event.repository.name }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Configure Git safe directory + run: git config --global --add safe.directory $PWD + - name: Pip install package + run: pip install -v . + - name: create_plugin_registries + run: create_plugin_registries + # Run unit tests for this repo if they exist + - name: Run pytest unit tests + run: | + echo "::group::pytest_unit_test" + echo "Pytest short unit tests of ${CURR_REPO}" + echo "which pytest" + which pytest + if [ ! -d "./tests/unit_tests" ]; then + echo "No unit tests found in ${CURR_REPO}" + exit 0 + fi + echo "pytest -q ./tests/unit_tests" + pytest -q ./tests/unit_tests + ret=$? + echo "Return code: ${ret}" + echo "::endgroup::" + if [[ "${ret}" != *"0"* ]]; then + echo "::error::due to geoips repo pytest errors, return code ${ret}" + exit 1 + fi + - name: Run base GeoIPS pytest unit tests + if: ${{ env.CURR_REPO != 'geoips' }} + run: | + echo "::group::pytest_unit_test" + echo "Pytest short unit tests of ${CURR_REPO} using core GeoIPS tests" + echo "which pytest" + which pytest + echo "pytest -q $BASE_GEOIPS_PATH/tests/unit_tests" + pytest -q $BASE_GEOIPS_PATH/tests/unit_tests + ret=$? + echo "Return code: ${ret}" + echo "::endgroup::" + if [[ "${ret}" != *"0"* ]]; then + echo "::error::due to geoips repo pytest errors, return code ${ret}" + exit 1 + fi diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 000000000..dc6066e2f --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,18 @@ +name: Lint + +# Note we want to run linting all the time +on: + # Triggers the workflow when pull request created and updated + pull_request: + # Allows run of this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + lint: + name: Lint + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-lint.yaml@main + permissions: + contents: read + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/new-brassy-note.yaml b/.github/workflows/new-brassy-note.yaml deleted file mode 100644 index a96c76a56..000000000 --- a/.github/workflows/new-brassy-note.yaml +++ /dev/null @@ -1,92 +0,0 @@ -name: Generate Release Note with Brassy - -on: - workflow_dispatch: - push: - tags: - - '*' - -jobs: - generate_release: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - - name: Extract branch name - shell: bash - run: | - git rev-parse --abbrev-ref HEAD - echo ${{ github.ref }} - export raw=$(git branch -r --contains ${{ github.ref }}) - echo $raw - branch=${raw##*/} - echo $branch - echo "branch=$branch" >> $GITHUB_OUTPUT - echo "Branch is $branch." - id: extract_branch - - - - name: Extract version - run: | - git fetch --quiet --tags origin # needed bc action above doesn't pull tags - export GEOIPS_VERS=`git -C . tag --sort=-creatordate | grep -v osr | head -n 1` - echo "version=$GEOIPS_VERS" >> $GITHUB_OUTPUT - echo "Version is $GEOIPS_VERS" - id: extract_version - - - name: Install Brassy - run: | - pip3 install brassy - - - name: Run Brassy on file - run: | - touch RELEASE_NOTE_HEADER - echo ".. dropdown:: Distribution Statement" >> RELEASE_NOTE_HEADER - echo " " >> RELEASE_NOTE_HEADER - echo " | # # # This source code is protected under the license referenced at" \ - >> RELEASE_NOTE_HEADER - echo " | # # # https://github.com/NRLMMD-GEOIPS." \ - >> RELEASE_NOTE_HEADER - - export output_file=v$(echo "${{ steps.extract_version.outputs.version }}" | tr . _).rst - echo $output_file - - brassy ./docs/source/releases/latest \ - --output-file ./docs/source/releases/$output_file \ - --prefix-file RELEASE_NOTE_HEADER \ - --no-rich \ - --release-version ${{ steps.extract_version.outputs.version }} - - rm RELEASE_NOTE_HEADER - - - name: Commit new release note and delete old yaml files - run: | - git config --global user.name 'GitHub Actions' - git config --global user.email 'github-actions@github.com' - git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY - git checkout -b \ - update-release-note-${{ steps.extract_branch.outputs.branch }} - git add ./docs/source/releases/*.rst - git commit -m "Add Built Release Note" - git rm ./docs/source/releases/latest/*.yaml - git commit -m "Remove old yaml files" - - - name: Push new release note to branch - run: | - git push --set-upstream origin \ - update-release-note-${{ steps.extract_branch.outputs.branch }} - - - name: create pull request - run: | - gh pr create \ - --base ${{ steps.extract_branch.outputs.branch }} \ - --head update-release-note-${{ steps.extract_branch.outputs.branch }} \ - --title \ - 'Merge new release note into ${{ steps.extract_branch.outputs.branch }}' \ - --body 'Created by manually run Github action.' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/package-and-publish.yaml b/.github/workflows/package-and-publish.yaml index 33c33a6a0..033274226 100644 --- a/.github/workflows/package-and-publish.yaml +++ b/.github/workflows/package-and-publish.yaml @@ -1,31 +1,38 @@ -name: Package and publish +name: Package/publish + +# 1. Brassy - on merge of v*-version-release +# * generate release notes +# * commit and push to v*-add-rst-release-note +# * open PR to default branch +# 2. Tag and Release - on merge of v*-add-rst-release-note +# * Tag current version +# * Release just tagged version +# 3. Package and Publish - on published release (from #2) +# * Build wheel +# * Publish to pypi +# 4. Deploy docs - on published release (from #2) +# * pip install geoips +# * pip install plugin repo +# * build docs with geoips/docs/build_docs.sh +# * deploy docs with geoips/docs/deploy_pages.sh on: - workflow_dispatch: + # triggers the workflow on published release release: - types: [published] + types: + - published + # allows run of this workflow manually from the actions tab + # must be merged to default before it will be available to manually run. + workflow_dispatch: jobs: - deploysdist: - name: "Deploy to Pypi" - runs-on: ${{ vars.RUNNER }} - - steps: - - name: Checkout source - uses: actions/checkout@v3 - with: - fetch-depth: 0 - fetch-tags: true - - - name: Create distributions - shell: bash -l {0} - run: | - python -m pip install -U build pip - python -m build - - name: Publish package to PyPI - # upload to PyPI on every release for a tag starting with 'v' - # if: github.event.action == 'published' && startsWith(github.event.release.tag_name, 'v') - uses: pypa/gh-action-pypi-publish@v1.8.10 - with: - user: ${{ secrets.PYPI_USER }} - password: ${{ secrets.PYPI_PASSWORD }} + package-publish: + name: Package/publish + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-package-and-publish.yaml@main + permissions: + contents: read + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + pypi_user: ${{ secrets.PYPI_USER }} + pypi_password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/proper-release-note-edits.yaml b/.github/workflows/proper-release-note-edits.yaml new file mode 100644 index 000000000..9b08747d4 --- /dev/null +++ b/.github/workflows/proper-release-note-edits.yaml @@ -0,0 +1,16 @@ +name: Note edits + +# We only need to check status of release notes from pull requests. +on: + # Triggers the workflow when pull request created and updated + pull_request: + # Allows run of this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + note-edits: + name: Note edits + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-proper-release-note-edits.yaml@main + permissions: + contents: read diff --git a/.github/workflows/tag-and-release.yaml b/.github/workflows/tag-and-release.yaml new file mode 100644 index 000000000..f8a1e2399 --- /dev/null +++ b/.github/workflows/tag-and-release.yaml @@ -0,0 +1,25 @@ +name: Tag/release + +on: + # Triggers the workflow when pull request closed (either closed + # as non-planned, or actually approved and merged.) + pull_request: + types: + - closed + branch: main + # Allows run of this workflow manually from the Actions tab + # Must be merged to default branch before it will be available + # to manually run. + workflow_dispatch: + +jobs: + tag-release: + name: Tag/release + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-tag-and-release.yaml@main + # Only run this if the pull request was merged, not just closed. + if: github.event.pull_request.merged == true + permissions: + contents: write + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows_archived/on-closed-pr.yaml b/.github/workflows_archived/on-closed-pr.yaml new file mode 100644 index 000000000..48d938bdc --- /dev/null +++ b/.github/workflows_archived/on-closed-pr.yaml @@ -0,0 +1,57 @@ +name: Call GeoIPS workflows triggered on PR close + +on: + # Triggers the workflow when pull request created and updated + pull_request: + workflow_dispatch: + +jobs: + print_run_flags: + runs-on: ${{ vars.RUNNER }} + steps: + - name: Print run flags + run: | + echo "github.ref_name ${{ github.ref_name }}" + echo "github.ref ${{ github.ref }}" + echo "github.base_ref ${{ github.base_ref }}" + echo "github.head_ref ${{ github.head_ref }}" + echo "vars.RUNNER ${{ vars.RUNNER }}" + echo "vars.RUNNER_DOCKERBUILD ${{ vars.RUNNER_DOCKERBUILD }}" + echo "vars.RUNNER_DOCKERRUN ${{ vars.RUNNER_DOCKERRUN }}" + echo "github.event.repository.default_branch ${{ github.event.repository.default_branch }}" + echo "github.event.repository.name ${{ github.event.repository.name }}" + echo "github.event.pull_request.merged ${{ github.event.pull_request.merged }}" + echo "github.run_id ${{ github.run_id }}" + echo "github.run_attempt ${{ github.run_attempt }}" + echo "vars.USE_CONDA_FOR_ACTIONS ${{ vars.USE_CONDA_FOR_ACTIONS }}" + echo "$GITHUB_CONTEXT" + echo ${{ github.context }} + + set_variables: + runs-on: ${{ vars.RUNNER }} + env: + CURR_REPO: ${{ github.event.repository.name }} + outputs: + # The name of the untagged docker images to be created by this workflow + pr_was_merged: ${{ steps.set_pr_status.outputs.PR_WAS_MERGED }} + steps: + - name: Set the PR status + id: set_pr_status + run: | + pr_was_merged=${{ github.event.pull_request.merged }} + echo "$pr_was_merged" + echo "PR_WAS_MERGED=$pr_was_merged" + echo "PR_WAS_MERGED=$pr_was_merged" >> $GITHUB_ENV + echo "PR_WAS_MERGED=$pr_was_merged" >> $GITHUB_OUTPUT + + call_on_closed_pr: + permissions: + pull-requests: write + contents: write + # You do not appear to be able to use variables in this "uses" field. + uses: NRLMMD-GEOIPS/.github/.github/workflows/on-closed-pr.yaml@main + needs: set_variables + with: + pr_was_merged: ${{ needs.set_variables.outputs.PR_WAS_MERGED }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 11ac4401d..2f690a1f6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,10 @@ profiling/ geoips/commandline/ancillary_info/cmd_instructions.json # auto-generated alias mapping file for the cli geoips/commandline/ancillary_info/alias_mapping.json -# auto-generated release note file for latest changes -docs/source/releases/latest.rst +# auto-generated release note files, all release note information +# is stored in .yaml format and the rst files auto-generated. +# index.rst, latest.rst, v*.rst +docs/source/releases/*.rst # auto-generated plugin registry file registered_plugins.json # auto-generated version file, created at run-time diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 726720a79..93be05f54 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,11 +7,3 @@ Please see https://github.com/NRLMMD-GEOIPS/geoips/blob/main/CHANGELOG_TEMPLATE.rst for instructions on updating release notes appropriately with each PR. - -The release note that is currently, actively being updated in -all geoips plugin repositories is referenced in the file: - -* https://github.com/NRLMMD-GEOIPS/geoips/blob/main/update_this_release_note - -* Please add all release notes for the current PR in the file referenced - within ``update_this_release note``. diff --git a/CHANGELOG_TEMPLATE.rst b/CHANGELOG_TEMPLATE.rst index 230033429..136719a65 100644 --- a/CHANGELOG_TEMPLATE.rst +++ b/CHANGELOG_TEMPLATE.rst @@ -3,338 +3,20 @@ | # # # This source code is protected under the license referenced at | # # # https://github.com/NRLMMD-GEOIPS. -:: - - #################################################################################### - DO NOT MODIFY THIS TEMPLATE! - FOR USE IN COMPILING CONSISTENT RELEASE NOTES! - - NOTE: RST IS VERY PARTICULAR ABOUT SPACING / INDENTATIONS! - FOLLOW THIS TEMPLATE CLOSELY TO ENSURE PROPER RENDERING (and read up on rst syntax) - #################################################################################### - -1. Follow below template when adding information to the current release note - prior to opening a pull request - - - Follow proper formatting/categorization of changes below - - - newest changes go at the top, just below distribution - statement. - - Ensure sections are in correct order, as listed below - - Follow spacing and indentation in template closely, - RST is very particular about spacing and indentation - (and read up on RST syntax for good measure). - - - If you make changes during the Pull Request process, just update - the release note entry accordingly, do not add an additional - release note entry. We only want the information regarding the final - merged pull request in the release notes. - -2. Only include headers for categories that have current updates (do NOT - include empty headers - if there are no "Breaking Changes" do not include - the "Breaking Changes" header) - -3. Include a list of files for each "Summary of change" section. - - - Simple changes can just have the list of modified files directly - under the “Summary of change” section, without additional details. - -4. Issue ID should match related linked issue, like: NRLMMD-GEOIPS/geoips#27 -5. Ensure your entries are included directly in the release note file: - docs/source/releases/vX_Y_Z.rst - - - NOTE: X.Y.Z is the UPCOMING VERSION of the repository, NOT the current - tagged version! - - The current release note to update directly on any geoips plugin - repository with current changes is referenced in the file - `update_this_release_note` within the geoips repo default branch: - https://github.com/NRLMMD-GEOIPS/geoips/blob/main/update_this_release_note - -:: - - ################################################################################### - NOTE: RST IS VERY PARTICULAR ABOUT SPACING / INDENTATIONS! - FOLLOW THIS TEMPLATE CLOSELY TO ENSURE PROPER RENDERING (and read up on rst syntax) - ################################################################################### - - -Version X.Y.Z (YYYY-MM-DD) -************************** - -* Very brief bulleted summary of changes -* Should reference major changes outlined below - -Breaking Changes -================ - - -------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* More info about the change - - * Additional optional information - -:: - - modified: /path/to/changed/file/related/to/breaking/change/1 - new: /path/to/new/file/related/to/breaking/change/1 - deleted: /path/to/deleted/file/related/to/breaking/change/1 - renamed: /path/to/orig/file -> /path/to/new/file - - -------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* More info about breaking Change 2 - - * Additional optional information - -* More info about breaking change 2 - -:: - - modified: /path/to/changed/file1/related/to/breaking/change/2 - modified: /path/to/changed/file2/related/to/breaking/change/2 - -Security Updates -================ - - ---------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* required security change due to file update - -:: - - modified: /paths/to/changed/file(s) - - -Major New Functionality -======================= - - -------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to new functionality - -:: - - modified: /paths/to/changed/file(s) - - -Enhancements -============ - - ------------------------------------------------ - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to improvements or enhancements to the code - -:: - - modified: /paths/to/changed/file(s) - - -Deprecations -============ - - ------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to code that has been or will be deprecated - -:: - - modified: /paths/to/changed/file(s) - - -Regression Fixes -================ - - ------------------------------------------------------------ - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to code that used to work in a previous release, - and is now broken. - -:: - - modified: /paths/to/changed/file(s) - - -Bug Fixes -========= - - ------------------------------------------ - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to bug fixes - -:: - - modified: /paths/to/changed/file(s) - - -Efficiency Improvements -======================= - - ------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to efficiency improvements - -:: - - modified: /paths/to/changed/file(s) - - -Installation Updates -==================== - - --------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to installation updates - -:: - - modified: /paths/to/changed/file(s) - - -Real-time Processing Updates -============================ - - ---------------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to real-time processing updates - -:: - - modified: /paths/to/changed/file(s) - - -Refactoring Updates -=================== - - ------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to code refactoring - -:: - - modified: /paths/to/changed/file(s) - - -Code Formatting and Style Updates -================================= - - ------------------------------------------------------------ - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to code formatting and style updates - -:: - - modified: /paths/to/changed/file(s) - - -GitHub Actions Updates -====================== - - ---------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to GitHub Actions updates - -:: - - modified: /paths/to/changed/file(s) - - -Git Workflow Improvements -========================= - - -------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to Git workflow improvements - -:: - - modified: /paths/to/changed/file(s) - -Test Repo Updates -================= - - --------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change to TEST REPO OUTPUT - -:: - - modified: /paths/to/changed/file(s) - -Testing Updates -=============== - - --------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change that WILL IMPACT TESTING - (or change to testing process) - -:: - - modified: /paths/to/changed/file(s) - - -Documentation Updates -===================== - - --------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change to documentation (contents, or build process) - -:: - - modified: /paths/to/changed/file(s) - +The GeoIPS Ecosystem uses ``brassy`` for automating RST release note generation from +user-generated YAML release notes. Every pull request within the GeoIPS GitHub +organization must have valid YAML release notes in the ``docs/source/releases/latest/`` +directory prior to being approved and merged. + +1. Run `brassy -t docs/source/releases/latest/.yaml` to create a template + release note that you can modify. +2. Edit brassy YAML release note template with all updates made on current branch. + + * Select appropriate category as found in the template, or create your own! + + * Empty sections in the YAML template can remain - they will be ignored. + * **Title** should be a short description of the change, < 120 characters, preferably <60 + * **Description** can be multiple lines (use proper YAML formatting), with additional + details. + * Always include a list of add/modified/deleted **files** for each entry. + * **Related Issue** ID should match related linked issue, like: NRLMMD-GEOIPS/geoips#27 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 0f6f2d247..bb34ea19f 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,3 +1,6 @@ + # # # This source code is protected under the license referenced at + # # # https://github.com/NRLMMD-GEOIPS. + # Contributor Covenant Code of Conduct This Code of Conduct is maintained and enforced by our community collaborators, diff --git a/LICENSE b/LICENSE index 92411ba4b..9ad9ee4bf 100644 --- a/LICENSE +++ b/LICENSE @@ -9,6 +9,7 @@ rights or licenses to the Computer Software are granted. Unauthorized use, sale, conveyance, disposition, or modification of a Computer Software may result in civil penalties and/or criminal penalties under 18 U.S.C. § 641. + 2. For purposes of this Agreement, the following definitions apply: a. “Computer software” means computer programs, source code, source code diff --git a/bandit.yml b/bandit.yml new file mode 100644 index 000000000..c316793bf --- /dev/null +++ b/bandit.yml @@ -0,0 +1,2 @@ +exclude: + - build/ diff --git a/docs/build_docs.sh b/docs/build_docs.sh index 0320b12e8..1a4809f66 100755 --- a/docs/build_docs.sh +++ b/docs/build_docs.sh @@ -105,12 +105,6 @@ if [[ "$4" != "" ]]; then else geoipsdocpath="$GEOIPS_PACKAGES_DIR/geoips/docs" fi -if [[ "$5" != "" ]]; then - echo "passed geoips_vers=$5" - geoips_vers=$5 -else - geoips_vers=latest -fi if [[ ! -d "$geoipsdocpath" ]]; then echo "***************************************************************************" echo "ERROR: GeoIPS docs path, with templates, does not exist:" @@ -166,54 +160,97 @@ fi # Since we revert index.rst at the end of this script, make sure the # user does not have any local modifications before starting. git_status_index=`git -C $docbasepath status docs/source/releases/index.rst` -if [[ "$git_status_index" == *"docs/source/releases/index.rst"* ]]; then +if [[ "$git_status_index" == *"docs/source/releases/"*".rst"* ]]; then echo "***************************************************************************" - echo "ERROR: Do not modify docs/source/releases/index.rst directly" - echo "Auto-generated within build_docs.sh using brassy." + echo "ERROR: Do not modify docs/source/releases/*.rst directly" + echo "All RST release files are auto-generated within build_docs.sh using brassy." echo "Please revert your changes and try again." echo "***************************************************************************" exit 1 fi -# Release notes are ALWAYS written in the "latest" folder, whether we are -# producing the generic "latest.rst" release note, or the specific -# vX_Y_Z.rst release note for an actual version release. -current_release_notes=`ls $docbasepath/docs/source/releases/latest/*` -# If we found any current YAML release notes, we need to generate the -# release RST file, and update index.rst with the current release version. -if [[ "$current_release_notes" != "" ]]; then - echo "Running brassy to generate current release note ${geoips_vers}.rst" +# Loop through all of the directories in docs/source/releases. +# Each set of YAML release notes is in a single directory, so +# we will build them all into RST release notes prior to building +# the docs. +for version_dir in $docbasepath/docs/source/releases/*/ ; do + echo $version_dir + # Only attempt to build release notes from a directory if there + # are YAML release notes in the directory. + notes_in_dir=`ls $version_dir/*.yaml` + if [[ "$notes_in_dir" == "" ]]; then + echo "SKIPPING: Empty directory $version_dir, skipping" + continue + fi + # Remove the trailing '/' from the version_dir (the way the + # directories were listed with /*/ above the trailing '/' + # is included in the path) + version=`basename ${version_dir::-1}` + # Explicitly skip the "upcoming" folder in the documentation. + # By the time we are ready to include these YAML release notes, + # they will be moved to the "latest" directory. "upcoming" only + # exists during the release process. + if [[ "$version" == "upcoming" ]]; then + echo "SKIPPING: Not including 'upcoming' version in documentation." + continue + fi + + # Create a temporary file with the header for the release note. + # This is passed into brassy as the prefix-file. + RELEASE_NOTE_HEADER=$docbasepath/docs/source/releases/RELEASE_NOTE_HEADER + touch $RELEASE_NOTE_HEADER + echo ".. dropdown:: Distribution Statement" >> $RELEASE_NOTE_HEADER + echo " " >> $RELEASE_NOTE_HEADER + echo " | This file is auto-generated. Please abide by the license found at" \ + >> $RELEASE_NOTE_HEADER + echo " | ${GEOIPS_REPO_URL}." \ + >> $RELEASE_NOTE_HEADER + + echo "Running brassy to generate current release note ${version}.rst" echo "" - echo "touch $docbasepath/docs/source/releases/${geoips_vers}.rst" - echo "brassy --release-version $geoips_vers --no-rich \" - echo " --output-file $docbasepath/docs/source/releases/${geoips_vers}.rst \" - echo " $docbasepath/docs/source/releases/latest" + echo "touch $docbasepath/docs/source/releases/${version}.rst" + echo "brassy --release-version $version --no-rich \ " + echo " --prefix-file $RELEASE_NOTE_HEADER \ " + echo " --output-file $docbasepath/docs/source/releases/${version}.rst \ " + echo " $docbasepath/docs/source/releases/$version" + echo "pink $docbasepath/docs/source/releases/${version}.rst" echo "" # Right now brassy does not auto-generate vers.rst, so we must touch it in # advance. - touch $docbasepath/docs/source/releases/${geoips_vers}.rst + touch $docbasepath/docs/source/releases/${version}.rst # Brassy creates the rst file! - brassy --release-version $geoips_vers --no-rich \ - --output-file $docbasepath/docs/source/releases/${geoips_vers}.rst \ - $docbasepath/docs/source/releases/latest - if [[ "$?" != "0" ]]; then + brassy --release-version $version --no-rich \ + --prefix-file $RELEASE_NOTE_HEADER \ + --output-file $docbasepath/docs/source/releases/${version}.rst \ + $docbasepath/docs/source/releases/${version} + brassy_retval=$? + # Remove the temporary release note header + rm -f $docbasepath/docs/source/releases/RELEASE_NOTE_HEADER + + # Pinken the RST file + pink $docbasepath/docs/source/releases/${version}.rst + pink_retval=$? + if [[ "$brassy_retval" != "0" || "$pink_retval" != "0" ]]; then # Exit here if brassy failed, because the doc build will subsequently fail. - echo "FAILED brassy ${geoips_vers}.rst release note generation failed." - echo "Please resolve release note formatting noted above and retry" - exit 1 - fi - # Ensure index.rst is updated with the latest release notes. - # This will eventually likely be rolled into brassy. - echo "Adding latest section to release note index" - echo "python $geoipsdocpath/update_release_note_index.py $docbasepath/docs/source/releases/index.rst $geoips_vers" - python $geoipsdocpath/update_release_note_index.py $docbasepath/docs/source/releases/index.rst $geoips_vers - if [[ "$?" != "0" ]]; then - # If this failed, exit here, because doc build will subsequently fail. - echo "FAILED update_release_note_index.py for version ${geoips_vers}" + echo "FAILED brassy ${version}.rst release note generation failed." echo "Please resolve release note formatting noted above and retry" + echo "brassy retval: $brassy_retval" + echo "pink retval: $pink_retval" exit 1 fi +done + +# Ensure index.rst is updated with the latest release notes. +# This will eventually likely be rolled into brassy. +echo "Creating docs/source/releases/index.rst with all release notes" +echo "python $geoipsdocpath/update_release_note_index.py $docbasepath/docs/source/releases/index.rst $docbasepath/docs/source/releases" +python $geoipsdocpath/update_release_note_index.py $docbasepath/docs/source/releases/index.rst $docbasepath/docs/source/releases +if [[ "$?" != "0" ]]; then + # If this failed, exit here, because doc build will subsequently fail. + echo "FAILED update_release_note_index.py." + echo "Please resolve release note formatting noted above and retry" + exit 1 fi buildfrom_docpath=$docbasepath/build/buildfrom_docs @@ -237,9 +274,9 @@ fi # below, and in that explicit order. # (directory name within docs/source, followed by heading name within index.rst # in parentheses below): -# * Required: introduction (Introduction) +# * Optional: introduction (Introduction) # * Optional: starter (Getting Started) -# * Required: userguide (User Guide) +# * Optional: userguide (User Guide) # * Optional: devguide (Developer Guide) # * Optional: deployment_guide (Deployment Guide) - note, NOT in geoips repo # * Optional: operator_guide (Operator Guide) - note, NOT in geoips repo @@ -249,24 +286,36 @@ fi # Plugin package documentation will follow the same order, only including the # sections included in their docs/source directory. -# Note api, introduction, and userguide are required for all plugin packages. +# Note api and releases are required for all plugin packages. # Those are hard coded in source/_templates/index_PKG.html -# starter, devguide, and contact are optional - if index.rst exist for those -# sections within a plugin package, include them in the main index. If they -# do not exist, don't include them. They are added to index_PKG.html template +# introduction, starter, userguide, devguide, and contact are optional - +# if index.rst exist for those sections within a plugin package, include +# them in the main index. If they do not exist, don't include them. +# They are added to index_PKG.html template # by search/replacing for *IDX in the template below. + +# API and releases are NOT included in these search/replace templates, +# since they are hard coded in index_PKG.html, so are required to be included. +introidx="introduction\/index" startidx="starter\/index" +userguideidx="userguide\/index" devguideidx="devguide\/index" deployguideidx="deployguide\/index" opguideidx="opguide\/index" contactidx="contact\/index" # Only include the following links in the index if they exist +if [[ ! -f $docbasepath/docs/source/introduction/index.rst ]]; then + introidx="" +fi if [[ ! -f $docbasepath/docs/source/deployguide/index.rst ]]; then deployguideidx="" fi if [[ ! -f $docbasepath/docs/source/opguide/index.rst ]]; then opguideidx="" fi +if [[ ! -f $docbasepath/docs/source/userguide/index.rst ]]; then + userguideidx="" +fi if [[ ! -f $docbasepath/docs/source/starter/index.rst ]]; then startidx="" fi @@ -323,15 +372,14 @@ if [[ "$pdf_required" == "True" ]]; then echo " try 'conda install latexcodec' if in anaconda" echo " or re-run with html_only to only build" echo " html documentation." - echo "" - echo "Reverting $docbasepath/docs/source/releases/index.rst" - git -C $docbasepath checkout docs/source/releases/index.rst exit 1 fi # do not include release notes in the PDF # Include starter, contact, devguide if they exist. releasidx="" sed "s/PKGNAME/${pkgname}/g; s/STARTERIDX/${startidx}/g; \ + s/INTROIDX/${introidx}/g; \ + s/USERGUIDEIDX/${userguideidx}/g; \ s/CONTACTIDX/${contactidx}/g; \ s/DEPLOYGUIDEIDX/${deployguideidx}/g; s/OPGUIDEIDX/${opguideidx}/g; \ s/DEVIDX/${devguideidx}/g; s/RELEASEIDX/${releasidx}/g;" \ @@ -385,6 +433,8 @@ if [[ "$html_required" == "True" ]]; then releasidx="releases\/index" # Include starter, contact, devguide if they exist. sed "s/PKGNAME/${pkgname}/g; s/STARTERIDX/${startidx}/g; \ + s/INTROIDX/${introidx}/g; \ + s/USERGUIDEIDX/${userguideidx}/g; \ s/CONTACTIDX/${contactidx}/g; \ s/DEPLOYGUIDEIDX/${deployguideidx}/g; s/OPGUIDEIDX/${opguideidx}/g; \ s/DEVIDX/${devguideidx}/g; s/RELEASEIDX/${releasidx}/g;" \ @@ -420,13 +470,6 @@ fi echo "" echo "***" -# docs/source/releases/index.rst should only be auto-generated, -# so revert the changes we just made. Note we checked at the beginning -# if this was already modified, and exited if there were any local modifications, -# to ensure the user had not manually modified it. -# git -C $docbasepath status docs/source/releases/index.rst -echo "Reverting $docbasepath/docs/source/releases/index.rst" -git -C $docbasepath checkout docs/source/releases/index.rst date -u echo "***" echo "" diff --git a/docs/dev/coding_standards.rst b/docs/dev/coding_standards.rst new file mode 100644 index 000000000..d77db74be --- /dev/null +++ b/docs/dev/coding_standards.rst @@ -0,0 +1,205 @@ +.. dropdown:: Distribution Statement + + | # # # This source code is protected under the license referenced at + | # # # https://github.com/NRLMMD-GEOIPS. + +================================= +🖍️ Coding Standards Reference Doc +================================= + +*"Although that way may not be obvious at first unless you're Dutch."* +- the Zen of Python + + +.. contents:: + +Summary +======= + +The goal is consistency. In style, we're in the right even if we're all wrong together. +GeoIPS code is meant to be re-used by many, and we expect that the code will be read +much more than it is written. The goal of this document is to help us spend less time +on conventions and more time making GeoIPS better. + +We use - in order of supremacy - a few Internal Standards, followed by the numpy +docstring standards and the python style guide laid out in PEP 8. + +This document will detail internal standard conventions and include some of the external +standards for easy reference. For external standards, the primary source is always +right in the case of a conflict with this document. + +External Style Standards +------------------------ + +`PEP 8 `__ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A nice version of PEP8 can be found at: `PEP8.org ` + +Some highlights below for reference. + +Function Names +"""""""""""""" + +Function names should be lowercase, with words separated by +underscores as necessary to improve readability. + +`PEP8 Names Standards `__ + +Class Names +""""""""""" + +Class names should normally use the CapWords convention. + +`PEP8 Names Standards `__ + +Exception Names +""""""""""""""" + +Because exceptions should be classes, the class naming convention applies here. +However, you should use the suffix "Error" on your exception names +(if the exception actually is an error). + +`PEP8 Names Standards `__ + +Module Names +"""""""""""" + +Modules should have **short**, **all-lowercase names**. +Underscores can be used in the module name if it improves readability. +`PEP8 Names Standards `__ + +Imports +^^^^^^^ +Imports should usually be on separate lines, e.g.: + +Yes: + +.. code-block:: python + + import os + import sys + +No: + +.. code-block:: python + + import os, sys + +It's okay to say this though: + +.. code-block:: python + + from subprocess import Popen, PIPE + +Imports are always put at the top of the file, just after any module comments and +docstrings, and before module globals and constants. + +`PEP8 Imports Standards `__ + +Internal Style Standards +------------------------- + +Bring code to standard in a dedicated PR +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We want to separate formatting/standardizing and functional changes to the code so +reviewing code is less painful. Please, if you're improving the functionality of code +and need to bring it to standard: + +1. Make a new branch (branch1) +2. Bring the code to standard +3. Open a PR and make a new branch from branch1 (branch2) +4. Make improvements to the functionality of the code on branch 2 +5. Open a second PR for branch 2 + +If easier, you can make the improvement before bringing the code to standard. + +If you touch code, it should meet standards +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We strongly recommend you update any functions you work on +if they do not meet the standard. At CIRA, this is a requirement for +PRs to be merged. For others, it's just a strong recommendation - +however, we don't want the burden of updating code to prevent you from contributing. +Please don't spend hours updating a 100,000 line module because you fixed a typo. +use discretion on when updates are needed. + +A good rule of thumb is that if you edit something and it doesn't have a docstring, +add it. If you edit more than 20% of a function/class/module, please edit the rest. + +Imports shouldn't be buried without a reason +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If an import needs to be buried for efficiency reasons or namespace conflicts, +this should be documented in the docstrings. + + +Linting/Formatting +^^^^^^^^^^^^^^^^^^ + +The GeoIPS project makes use of several linting tools to help maintain code quality. The +full suite of linters can be installed by installing the "test" dependencies via pip. +For example, if you installed GeoIPS using `pip install .` the linters can be installed +using `pip install .[test]` the following tools to ensure code quality: + +Black +^^^^^ + +We use the `Black formatter `_ with its default +settings. As stated in the Black documentation, it is an uncompromizing code +formatter, but it has resulted in significantly more readable code. Applying it +automatically while writing code has also reduced development time since +developers don't need to think about formatting. + +Flake8 +^^^^^^ + +We use the `Flake8 linter `_ to enforce +PEP8 code standards. We also add several plugins to Flake8 to enforce additional +standards for GeoIPS code. Plugins used include: + +- `flake8-docstrings `_ is used to enforce + the numpy docstring standard. +- `flake8-rst-docstrings `_ is + used to ensure that docstrings are valid reStructuredText. +- `flake8-rst ` runs flake8 on code + snippets in reStructuredText files to ensure proper formatting in + documentation. + +We modify the default behavior of flake8 slightly to make it work well with Black, +ignore specific errors, and configure plugins. GeoIPS specific settings for +flake8 include the following: + +```toml +[flake8] +max-line-length=88 +count=True +ignore=E203,W503,E712 +extend-exclude=_version.py,lib,*_docs,geoips_dev_utils +docstring-convention=numpy +rst-roles=class,func,ref +rst-directives=envvar,exception +rst-substitutions=version +statistics=True +per-file-ignores = + /*/interfaces/__init__.py:F401 +``` + +Github Conventions +------------------ + +Pull Request Workflow +^^^^^^^^^^^^^^^^^^^^^ + +`https://nrlmmd-geoips.github.io/geoips/devguide/git_workflow.html#geoips-github-pull-request-workflow `__ + +`https://nrlmmd-geoips.github.io/geoips/devguide/git_workflow.html#geoips-merge-pr-and-close-issue-workflow `__ + +Issue Workflow +^^^^^^^^^^^^^^ + +`https://nrlmmd-geoips.github.io/geoips/devguide/git_workflow.html#geoips-github-issue-creation-workflow `__ + +Other Conventions +----------------- diff --git a/docs/source/_templates/index_PKG.html b/docs/source/_templates/index_PKG.html index 6867868e7..ac8f6988c 100644 --- a/docs/source/_templates/index_PKG.html +++ b/docs/source/_templates/index_PKG.html @@ -11,7 +11,9 @@ **Date**: |today| **Version**: |release| -**Download PDF documentation**: :download:`GeoIPS_PKGNAME.pdf` +.. + Temporarily comment out PDF documentation download. PDF build fails. + **Download PDF documentation**: :download:`GeoIPS_PKGNAME.pdf` **Previous versions**: Documentation of previous PKGNAME versions are available at `github.com/NRLMMD-GEOIPS `__. @@ -19,23 +21,16 @@ **Useful links**: `Source Repository `__ | `GeoIPS License `__ | -`NRLMMD `__ | +`NRLMMD `__ | -:mod:`PKGNAME` is a free software program, United States Government NRLMMD licensed. +:mod:`PKGNAME` is a free software program, using the NRL Open License Agreement. :: Distribution Statement A. Approved for public release. Distribution is unlimited. - Author: - Naval Research Laboratory, Marine Meteorology Division - - This program is free software: you can redistribute it and/or modify it under - the terms of the NRLMMD License included with this program. This program is - distributed WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the included license - for more details. If you did not receive the license, for more information see: - https://github.com/U-S-NRL-Marine-Meteorology-Division/ + This source code is protected under the license referenced at + https://github.com/NRLMMD-GEOIPS. .. automodule:: PKGNAME :noindex: @@ -122,9 +117,9 @@ :titlesonly: {% endif %} {% if not single_doc %} - introduction/index + INTROIDX STARTERIDX - userguide/index + USERGUIDEIDX DEVIDX DEPLOYGUIDEIDX OPGUIDEIDX diff --git a/docs/source/devguide/git_workflow.rst b/docs/source/devguide/git_workflow.rst index 2af6ee523..10ef4c72e 100644 --- a/docs/source/devguide/git_workflow.rst +++ b/docs/source/devguide/git_workflow.rst @@ -57,7 +57,7 @@ FROM WEB: MEMBERS: Create Branch from Existing Issue ==================================================== **NOTE: Those who are NOT members of the GeoIPS organization will fork,** -**not branch. Skip to NON MEMBERS section** +**not branch. If you are unable to branch, skip to NON MEMBERS section** * Navigate to Issue you would like to resolve * Click on Development->Create Branch @@ -68,15 +68,15 @@ FROM WEB: MEMBERS: Create Branch from Existing Issue * NOTE you can create branches on repositories outside the repository the Issue resides in - * **Change branch source** optional (defaults to "main") + * **Change branch source** optional (defaults to default branch) * Select **"Checkout locally"** * Click **"Create branch"** * Copy and paste the resulting "git fetch" and "git checkout" commands FROM WEB: NON-MEMBERS: Create fork of repo ========================================== -* **NOTE: NRLMMD-GEOIPS members will branch following steps above, - **skip this section if you are a member** +* **NOTE: GeoIPS organization members will branch following steps above, + **skip this section if you are a member and are able to branch** * Navigate to desired repository * Click drop down next to "Fork" * Click "+ Create a new fork" @@ -88,22 +88,22 @@ FROM WEB: NON-MEMBERS: Create fork of repo GeoIPS command line workflow **************************** -FROM COMMAND LINE: Switch to new branch, Make changes as usual -============================================================== +FROM COMMAND LINE: Switch to new branch/fork, Make changes as usual +=================================================================== * Navigate to repository of your choice * Issue only needs to be created on a single repository - * You can create branches and make changes on any number of repos, - as appropriate. + * You can create branches/forks and make changes on any number of + repos, as appropriate. * Related changes on different repositories will all be linked to the same Issue. -* Switch to new branch, and make changes as appropriate +* Switch to new branch/fork, and make changes as appropriate * *Ensure you copy and paste git fetch and git checkout commands* - *when creating branch above* - * Switch to new branch: Paste git fetch / git checkout commands + *when creating branch/fork above* + * Switch to new branch/fork: Paste git fetch / git checkout commands specified when creating branch from Issue * git fetch origin @@ -113,8 +113,8 @@ FROM COMMAND LINE: Switch to new branch, Make changes as usual * Use enforced commit message format for all commits - * Please follow - `Commit Message Template `_ + * Please follow `Commit Message Template + `_ * Summary line <= 120 characters * Blank line (if commit message is more than one line) * OPTIONAL: additional details @@ -124,7 +124,8 @@ FROM COMMAND LINE: Switch to new branch, Make changes as usual * Before pushing your final changes to GitHub and creating a pull request, you MUST update the release notes with your current changes. - * Instructions found in `CHANGELOG.rst `_ + * Instructions found in `CHANGELOG.rst + `_ * If release notes are not updated properly, pull request will not be approved. * Create test scripts and associated outputs for any new functionality @@ -147,8 +148,8 @@ Push changes to github GeoIPS GitHub Pull Request workflow *********************************** -FROM WEB: Create pull request from new ticket branch to "dev" branch -==================================================================== +FROM WEB: Create pull request from new ticket branch to default branch +====================================================================== Follow these instructions for each repo that requires changes for a given Issue. diff --git a/docs/source/introduction/index.rst b/docs/source/introduction/index.rst index e019cb71c..ba6fa4ee5 100644 --- a/docs/source/introduction/index.rst +++ b/docs/source/introduction/index.rst @@ -6,8 +6,6 @@ Introduction ************ -:ref:`requirement_spec` - .. toctree:: :maxdepth: 2 diff --git a/docs/source/new-docs/getting-started/best-practices/best_practices.rst b/docs/source/new-docs/getting-started/best-practices/best_practices.rst new file mode 100644 index 000000000..fc064a701 --- /dev/null +++ b/docs/source/new-docs/getting-started/best-practices/best_practices.rst @@ -0,0 +1,210 @@ +:orphan: + +.. dropdown:: Distribution Statement + + | # # # This source code is protected under the license referenced at + | # # # https://github.com/NRLMMD-GEOIPS. + +.. _best_practices: + +GeoIPS Generalized Best Practices +================================= + +There are many ways one can incorporate GeoIPS into their research or operational flow. +While the developers do not want to be prescriptivist, we do recommend a series +of best practices to ensure that your plugin development/deployment experience is as smooth as possible. +First, we will propose some generalized best practices to consider before developing a plugin, +followed by best practices to follow during development, during initial release/deployment, +and finally at the planned or unplanned EOL of your plugin. Future iterations of this document +will include best practices for operational and research use of GeoIPS. + +Best Practices at Conception of your Plugin +------------------------------------------- + +In general, try to leverage existing GeoIPS capabilities before developing your own plugins! + +* One of the core functions of GeoIPS is to eliminate redundant development time. Leverage that! +* If a reader exists, but is missing a core functionality required for your development, consider the + following options: + + * Reach out to the developer of the reader and request they add the feature(s) you are missing. Odds + are good that if you need it, someone else probably will too! + * Pull the plugin containing the reader and add the functionality yourself. Then, open a PR on the repo + containing the plugin to merge your changes. + * If all else fails and the reader is no longer actively maintained, create a local copy for your plugin + and provide proper attribution somewhere in your repository (provided you're going to open source your + plugin, of course.) + +Try to make an effort to attend the weekly open-source meetings routinely and become an active +participant in the discussions! As a community-driven effort, GeoIPS thrives as more people +join in! + +* This will ensure you're up to date on the latest developments and are aware of what's coming down + the pipeline. See the first bullet point for why that's important! +* You can also make requests of folks that are developing plugins and products to consider sources + you are interested in working with. + + * This means you may not have to do as much development work on any plugins you develop + * This also works both ways-- you might be able to save a member of the community some effort, + as well as drive some collaboration! +* You'll be able to provide guidance on the direction GeoIPS is heading, ensuring that it is robust + and has broad spectrum utility. + + * One prominent example includes the efforts of Tim Olander and Tony Wimmers's provision of + feedback during development of the plugin system. Their feedback helped shape our overall + implementation of plugins, and included one of the first demonstrations of the system via ARCHER. + * Another example includes the development efforts at CIRA by Evan Rose and Jeremy Solbrig, who, + while working closely with the community, have helped shape the way users interact with GeoIPS + from the lowest levels, to the highest levels. +* Lastly, you can share your success stories and drive interest in the plugins you are developing. + +Best Practices During Development of your Plugin +------------------------------------------------ + +When you develop a plugin, it is best to BEGIN by following the coding and formatting standards +recommended by the GeoIPS developers to facilitate easy sharing and integration into the GeoIPS +infrastructure. + +* Make sure that you're developing in your own repository and not on the GeoIPS main branch. +* Ensure your code is properly commented and legible for fellow developers. You never know how many + emails you'll be able to prevent by explaining your logic in advance! +* Make sure you follow the standards and practices recommended for providing informative runtime + messages, and that they are logged at the correct levels. + + * Be sure to include a healthy mix of INFO, INTERACTIVE, DEBUG, WARNING and ERROR-level messages! +* Consider running an automated system to monitor your code for formatting and potential security + risks (e.g., SQL injection.) + + * check_code.sh is a GeoIPS built-in method for monitoring these! + * This is especially crucial for your code to enter operations on controllsed systems e.g., FNMOC, + NAVO. + +Packages change frequently! Make sure you're always on the latest working branch of GeoIPS and +any plugins you are using before sharing your code! + +* Sometimes interfaces are deprecated between versions-- it's easier for everyone if you make sure + everything works on the latest version. + + * An upcoming GeoIPS 2.0 will retain backward functionality for most plugins, however we reserve + the right to revoke that capability at any time. +* To that end, stay abreast of upcoming deprecations within non-GeoIPS packages used by your plugin. + You never know when NumPy will finally make good on all those warnings you've been ignoring! + +We strongly discourage developing new, core infrastructure for GeoIPS, including new interfaces +and subsystems. + +* These kinds of developments have a strong chance of creating conflicts later on down the line that + may be difficult to resolve against the main GeoIPS repository. +* If you identify a need for new, core functionality raise your concern/request in the weekly meetings. + At best we may have an alternative option to recommend, and at worst you may be conscripted into + developing the feature! + +If your GeoIPS plugin contains ANY proprietary IP or other sensitive data, it is CRUCIALLY IMPORTANT +that you do not make any changes to the codebases of the main GeoIPS repository or any plugins you +are using! + +* In general, we will reject most commits made the main GeoIPS repository +* If that chance exists that your repo will enter open source, ensure that you are not committing + anything private or dangerous. + + * Check for things like private keys in the commits, small pieces of information that when + combined could lead to spillage, nasty merge comments, etc. + +Ensure that your code is performant and reasonably Pythonic. Nobody wants their code to be the +bottleneck in someone else's workflow! + +* If your plugin includes C or Fortran-wrapped algorithms that's also okay! We support interfaces + that use f2py, etc. You might consider converting to native Python later on down the line, but + that's never going to be mandatory. + +Include test scripts and archive the mininmum number of files required to test your plugin! + +* Doing so provides your users a quick way to understand how your plugin is invoked. + + * We recommend passing explicit file inputs to the procflow rather than a directory to ensure + exact testing. +* The ability of developers to validate their installation of your plugin is crucial. + + * To that end, consider including the final output images to be used in the + +Lastly, share your successes and struggles at the weekly open source meeting! + +* If you're running into problems or have questions, this is one of the best ways to resolve them. + + * You can also join the GeoIPS slack to ask questions 24/7 +* People may have good suggestions or more performant ways to accomplish your goals. +* People will love to see the pretty pictures you make. + +Best Practices Before and After Deploying your Plugin +----------------------------------------------------- + +While we can't control how your administrate your own repositories, or how you run +GeoIPS, we can make some suggestions! + +When you commit to an internal or public-facing repository, make sure ownership and permissions are +controlled appropriately. + +* Try to not let people freely merge code into your repository as it creates a security risk for + you and anyone using your plugin. +* Establish review requirements early on, and ensure that commits and PRs are fully tested before + being merged. Optionally (but ideally,) you'd test these on a fresh GeoIPS installation to catch + any dependencies that may be missing. + + * This is a good time to again ensure you're developing on the most recent versions of the GeoIPS + and plugins. +* Include documentation for your plugin! + + * You can use this repository's documentation as a standard. + +The infrastructure for automated processing and scheduling of GeoIPS runs on your system is +outside the scope this documentation. + +* We don't want to risk pigeonholing anyone or exposing our own internal systems. +* That said, in a future release we may make some basic repositories with simple database + and automation packages available. +* In general, though, all you need is way to identify when new files are available and a way + to automate the passing of those files to GeoIPS. There are myriad solutions from this ranging + from simple (cron) to advanced (Cylc) + +Let people know when your plugin is ready for release! + +* Share the good news at the open source meetings! Give talks at conferences! Advertise as + best as you can! + + * This is good for GeoIPS and good for you. You may wind up roping in new talent or find + transition avenues you weren't aware of. + +Finally, be reasonably available to support your plugin for end users. + +* Try to be an active participant on your repository's issues board at minimum. +* Answer questions that people have and address issues they may be coming across. +* Communicate your planned update schedule (if any) and try to stick to it. + + * If you plan to deprecate features of your plugin or bring it to end-of-life (EOL,) + be sure to announce that well in advance! + +Best Practices for Deprecation/EOL of your Plugin +------------------------------------------------- + +When your plugin crosses the rainbow bridge, due to loss of funding, personnel, or interest +consider the following: + +* Announce the upcoming deprecation/EOL at the GeoIPS weekly open source meeting. +* Include in your plugin repository a notice of upcoming EOL. +* Establish a chain of ownership/maintenance if possible. + + * Have in place a method for another developer to take control of the repository + for ongoing updates + * Remove or automatically redirect the repository to one where active development + of the plugin is ongoing + * Failing the above, place a prominent warning banning that indicates active + development of the plugin has ceased. + +GeoIPS Best Practices for Operations +====================================== +[COMING SOON] + +GeoIPS Best Practices for Research +====================================== +[COMING SOON] diff --git a/docs/source/releases/1.14.0/1.14.0.yaml b/docs/source/releases/1.14.0/1.14.0.yaml new file mode 100644 index 000000000..3035eb301 --- /dev/null +++ b/docs/source/releases/1.14.0/1.14.0.yaml @@ -0,0 +1,79 @@ +Installation: +- title: 'Remove PATH and PYTHON path from setup configs' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + This was causing problems with using the wrong python/pip during processing. + Do not explicitly set PATH or PYTHONPATH. + files: + added: + - '' + deleted: + - '' + modified: + - 'setup/config_geoips' + - 'setup/geoips_conda_init_setup' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' +GitHub Actions: +- title: 'Turn off pdf doc build from doc-test' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + We still need to fix the pdf doc build, but turning it off temporarily until + we determine what the issue is. + files: + added: + - '' + deleted: + - '' + modified: + - '.github/workflows/doc-test.yaml' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' +- title: 'Turn off RST release note check from version release branches' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + Since version releases often have corrections to RST release notes, do not fail + on RST release note check within version release branches. + files: + added: + - '' + deleted: + - '' + modified: + - '.github/workflows/release-note-check.yaml' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' +- title: 'Move 1.14.0 and 1.14.1 rst files into latest/*.yaml and upcoming/*.yaml' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + No longer deleting YAML files or checking in rst files - store + all release notes as YAML in perpetuity, and auto-generate the + rst with brassy every time. This will make rolling back releases + MUCH easier. + files: + added: + - '' + deleted: + - 'docs/source/releases/v1_14_0.rst' + - 'docs/source/releases/v1_14_1.rst' + modified: + - 'docs/source/releases/latest/v1_14_0.yaml' + - 'docs/source/releases/upcoming/v1_14_1.yaml' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' \ No newline at end of file diff --git a/docs/source/releases/1.14.0/1.14.0_environment_files.yaml b/docs/source/releases/1.14.0/1.14.0_environment_files.yaml new file mode 100644 index 000000000..8df16efc9 --- /dev/null +++ b/docs/source/releases/1.14.0/1.14.0_environment_files.yaml @@ -0,0 +1,25 @@ +Release Process: + - title: "Add pip and mamba 1.14.0 environment files" + description: | + *From GEOIPS#562: 2024-09-17, 1.14.0 updates* + + Package: geoips_full + Total run time: 2558 seconds + Number data types run: 56 + Number data types failed: 0 + Mon Sep 16 22:06:05 UTC 2024 + + Package: geoips_base + Total run time: 156 seconds + Number data types run: 3 + Number data types failed: 0 + Tue Sep 17 01:45:03 UTC 2024 + files: + added: + - environments/mamba_base_package_list_1.14.0_20240915.yml + - environments/mamba_full_package_list_1.14.0_20240916.yml + - environments/pip_base_requirements_1.14.0_20240915.txt + - environments/pip_full_requirements_1.14.0_20240916.txt + related-issue: + number: 562 + repo_url: "GEOIPS/geoips" diff --git a/docs/source/releases/1.14.0/build-all-rst-release-notes-every-time.yaml b/docs/source/releases/1.14.0/build-all-rst-release-notes-every-time.yaml new file mode 100644 index 000000000..bb54f6ee5 --- /dev/null +++ b/docs/source/releases/1.14.0/build-all-rst-release-notes-every-time.yaml @@ -0,0 +1,38 @@ +Documentation: +- title: 'Build RST release notes from YAML files every time' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + Rather than generating and checking in RST files with + each release, save all the original YAML files and produce + RST every time from within build_docs.sh. + + Store the release notes in X.Y.Z directories, producing + X.Y.Z.rst during the build_docs process. Update the index.rst + generating code to support either vX_Y_Z.rst or X.Y.Z.rst + filename format. + + This allows simply moving the 'latest' release note directory + to X.Y.Z during the release process, and maintaining a record + of all the original release notes in perpetuity, facilitating + rapid release roll backs or evaluation of previous release notes. + + Note we also now ignore an 'upcoming' release note directory, + as that is a temporary location to store the release note for + the upcoming release during the version release process for + the current version ('latest'). At the end of the release + process, the 'upcoming' directory will be moved to 'latest' + to prepare for the next release. + files: + added: + - '' + deleted: + - '' + modified: + - 'docs/build_docs.sh' + - 'docs/update_release_note_index.py' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' diff --git a/docs/source/releases/1.14.0/fix-pdf-doc-build.yaml b/docs/source/releases/1.14.0/fix-pdf-doc-build.yaml new file mode 100644 index 000000000..e8ce13b92 --- /dev/null +++ b/docs/source/releases/1.14.0/fix-pdf-doc-build.yaml @@ -0,0 +1,25 @@ +documentation: +- title: 'Resolve pdf doc build issues' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + PDF doc builds often fail when html succeeds - this MUST be tested regularly + on PR merge, to ensure we keep up with pdf builds as well as html. Much easier to + resolve issues as they crop up rather than much later when we realize it's failing. + + Temporarily turn off all pdf builds, and comment out the pdf download from the + documentation index template. + files: + added: + - '' + deleted: + - '' + modified: + - '.github/workflows/doc-test.yaml' + - '.github/workflows/reusable-deploy-docs.yaml' + - 'docs/source/_templates/index_PKG.html' + moved: + - '' + related-issue: + number: 562 + repo_url: '' diff --git a/docs/source/releases/1.14.0/make_all_doc_sections_optional.yaml b/docs/source/releases/1.14.0/make_all_doc_sections_optional.yaml new file mode 100644 index 000000000..cdbc05d6b --- /dev/null +++ b/docs/source/releases/1.14.0/make_all_doc_sections_optional.yaml @@ -0,0 +1,21 @@ +documentation: +- title: 'Make all documentation sections optional' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + Previously required introduction and user guide - allow building documentation + with only release notes and API. + files: + added: + - '' + deleted: + - '' + modified: + - 'docs/build_docs.sh' + - 'docs/source/_templates/index_PKG.html' + moved: + - '' + related-issue: + number: 562 + repo_url: '' + diff --git a/docs/source/releases/1.14.0/move-reusable-workflows-and-sync-to-geoips_ci.yaml b/docs/source/releases/1.14.0/move-reusable-workflows-and-sync-to-geoips_ci.yaml new file mode 100644 index 000000000..207e2af97 --- /dev/null +++ b/docs/source/releases/1.14.0/move-reusable-workflows-and-sync-to-geoips_ci.yaml @@ -0,0 +1,57 @@ +GitHub Actions: +- title: 'Move reusable workflows and sync files to geoips_ci' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + Create a new repository, geoips_ci, that will manage all GeoIPS Ecosystem + reusable workflows, as well as files to sync to all GeoIPS plugin + repositories. + + Simplify brassy-notes and tag-and-release to filter on pr merged immediately + (rather than trying to pass pr merged flag into reusable workflow). These should + never run on a closed but not merged PR. + files: + added: + - .github/workflows/brassy-notes.yaml + - .github/workflows/deploy-docs.yaml + - .github/workflows/package-and-publish.yaml + - .github/workflows/tag-and-release.yaml + deleted: + - .github/sync/.flake8 + - .github/sync/.github/release.yml + - .github/sync/.github/workflows/codeql.yml + - .github/sync/.gitignore + - .github/sync/CHANGELOG.rst + - .github/workflows/reusable-brassy-notes.yaml + - .github/workflows/reusable-check-release-branch.yaml + - .github/workflows/reusable-deploy-docs.yaml + - .github/workflows/reusable-get-versions.yaml + - .github/workflows/reusable-package-and-publish.yaml + - .github/workflows/reusable-print-variables.yaml + - .github/workflows/reusable-tag-and-release.yaml + modified: + - '' + moved: + - '' + related-issue: + number: 562 + repo_url: '' +- title: 'Add standard synced files to geoips repo' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + These files auto-copied from geoips_ci. + files: + added: + - .flake8 + - pytest.ini + - bandit.yml + deleted: + - '' + modified: + - 'CHANGELOG.rst' + moved: + - '' + related-issue: + number: 562 + repo_url: '' diff --git a/docs/source/releases/1.14.1/1.14.1-release-note.yaml b/docs/source/releases/1.14.1/1.14.1-release-note.yaml new file mode 100644 index 000000000..189ffba78 --- /dev/null +++ b/docs/source/releases/1.14.1/1.14.1-release-note.yaml @@ -0,0 +1,26 @@ +documentation: + - title: Add release note for version 1.14.1 + description: | + *From GEOIPS#562: 2024-09-24, 1.14.1 updates* + Release note on release branch to force tag/release, package/publish, + and deploy docs. + files: + added: + - docs/source/releases/latest/1.14.1-release-note.yaml + related-issue: + number: 562 + repo_url: GEOIPS +documentation: + - title: Update upcoming and tagged versions + description: | + *From GEOIPS#562: 2024-09-25, 1.14.1 updates* + + * Tagged version: 1.14.1 + * Upcoming version: 1.14.2 + files: + modified: + - .github/versions/tagged_version + - .github/versions/upcoming_version + related-issue: + number: 562 + repo_url: GEOIPS diff --git a/docs/source/releases/1.14.1/1.14.1.yaml b/docs/source/releases/1.14.1/1.14.1.yaml new file mode 100644 index 000000000..67daaaae7 --- /dev/null +++ b/docs/source/releases/1.14.1/1.14.1.yaml @@ -0,0 +1,19 @@ +Release Process: +- title: 'Update tagged_version and upcoming_version' + description: | + *From GEOIPS#562: 2024-08-13, 1.13.2 updates* + + Currently tagged version 1.14.0, upcoming version 1.14.1. + files: + added: + - '' + deleted: + - '' + modified: + - '.github/versions/tagged_version' + - '.github/versions/upcoming_version' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' diff --git a/docs/source/releases/index.rst b/docs/source/releases/index.rst deleted file mode 100644 index e7d51daf0..000000000 --- a/docs/source/releases/index.rst +++ /dev/null @@ -1,232 +0,0 @@ -.. dropdown:: Distribution Statement - - | # # # This source code is protected under the license referenced at - | # # # https://github.com/NRLMMD-GEOIPS. - -.. _release_notes: - -Release Notes -************* - -Version 1.14 ------------- - -.. toctree:: - :maxdepth: 1 - - v1_14_0a0 - -Version 1.13 ------------- - -.. toctree:: - :maxdepth: 1 - - v1_13_1 - v1_13_0 - v1_13_0a0 - -Version 1.12 ------------- - -Added documentation and enhancements for coding standards, fixed bugs in polar data, log functions, and Dockerfile; -Introduced documentation for PEP8, linters, and plugin registries; Refactored optional imports, plugin registries. Fixed -issues with create_plugin_registries, ATMS platform names, and pytest dependencies. - -.. toctree:: - :maxdepth: 1 - - v1_12_4 - v1_12_3 - v1_12_2 - v1_12_2a0 - v1_12_1 - v1_12_0 - -Version 1.11 ------------- - -Removed flake8_docstring_only flag from test_all.sh and full_test.sh across the repository. Fixed issues in integration -tests and ensured full flake8 tests applied. - -.. toctree:: - :maxdepth: 1 - - v1_11_7 - v1_11_7a0 - v1_11_6 - v1_11_5 - v1_11_5a0 - v1_11_3 - v1_11_3a0 - v1_11_2 - v1_11_1 - v1_11_1a0 - v1_11_0 - -Version 1.10 ------------- - -Updated interface names throughout the code base, moved YAML sector files to new directories within the geoips/plugins -structure, added support for both the old and new dynamic sector YAML formatting, and renamed plugins to enhance -clarity. Introduced a try/except block in sector_utils/tc_tracks.py to support both old and new dynamic sector YAML -formats. Consolidated both dynamic and static sectors under the unified directory structure in -geoips/plugins/yaml/sectors, and renamed various modules to reflect their functionalities more accurately. - -.. toctree:: - :maxdepth: 1 - - v1_10_3 - v1_10_2 - v1_10_1 - v1_10_0 - v1_10_0a13 - v1_10_0a12 - v1_10_0a11 - v1_10_0a10 - v1_10_0a9 - v1_10_0a8 - v1_10_0a7 - v1_10_0a6 - v1_10_0a5 - v1_10_0a4 - v1_10_0a3 - v1_10_0a2 - v1_10_0a1 - v1_10_0a0 - -Version 1.9 ------------ - -Removed GEOIPS BASEDIR references and introduced GEOIPS_TESTDATA_DIR and GEOIPS_DEPENDENCIES_DIR in -filenames/base_paths.py, updated and implemented sphinx-apidoc for Sphinx documentation builds, finalized RST -documentation outline to align with SharePoint documentation, replaced setup.py with pyproject.toml for project setup, -added "cbar_tick_labels" and "cbar_spacing" fields in Rain colormap mpl_color_info, allowed processing when no tropical -cyclones are present in the database, installed geoips[doc, test, lint] dependencies in setup.sh, and configured code -style to ignore flake8 W503 due to conflicts with black. - -.. toctree:: - :maxdepth: 1 - - v1_9_2 - v1_9_1 - v1_9_0 - -Version 1.8 ------------ - -Renamed SSMI/S 89GHz products to 91GHz, updated F-16 SSMI/S scaling, resolved basename error, added geoips_clavrx to -test_full_install, and corrected minor sphinx build warnings. Fixed tests, ensured accurate product naming, and improved -installation and documentation processes. - -.. toctree:: - :maxdepth: 1 - - v1_8_1 - v1_8_0 - -Version 1.7 ------------ - -Updated matplotlib version dependency, extended SAR incident angle range, and converted CHANGELOG and documentation -formats to RST. Removed outdated sections, fixed minor documentation issues, and improved documentation organization and -building process. - -.. toctree:: - :maxdepth: 1 - - v1_7_0 - -Version 1.6 ------------ - -Added resource usage statistics, documentation updates, and code style enforcement checks. Fixed numpy builtin aliases, -product family typos, and output requirements. - -.. toctree:: - :maxdepth: 1 - - v1_6_3 - v1_6_1 - v1_6_0 - -Version 1.5 ------------ - -Updated the Python installation process, added alternate command line arguments functions, created unique storm -directory names for invests, and fixed bugs. Expanded test repository updates and improved metadata handling. - -.. toctree:: - :maxdepth: 1 - - v1_5_3 - v1_5_2 - v1_5_1 - v1_5_0 - -Version 1.4 ------------ - -Updated documentation and installation requirements, added new test scripts and functionality. Fixed real-time -processing and coverage checks. - -.. toctree:: - :maxdepth: 1 - - v1_4_8 - v1_4_7 - v1_4_6 - v1_4_5 - v1_4_4 - v1_4_3 - v1_4_2 - v1_4_1 - v1_4_0 - -Version 1.3 ------------ - -Replaced instances of 'atcf' with 'tc', removed support for GEOIPSFINAL and GEOIPSTEMP, and consolidated directories -into 'geoips_outdirs.' Added dev/title.py interface, updated template pull request ticket, allowed passing -'title_copyright,' and added new DATABASESUCCESS and DATABASEFAILURE string checks. - -.. toctree:: - :maxdepth: 1 - - v1_3_2 - v1_3_1 - v1_3_0 - -Version 1.2 ------------ - -Updated bdeck parser, SMOS text winds, test_interfaces.py, attribute names, call signatures; added ABI test scripts, -Software Requirements Specification. Replaced original_source_filename with original_source_filenames; simplified setup, -command line args checking, installation steps. - -.. toctree:: - :maxdepth: 1 - - v1_2_5 - v1_2_4 - v1_2_3 - v1_2_2 - v1_2_1 - v1_2_0 - -Version 1.1 ------------ - -Addition of new features like MODIS reader, stitched product output, sector overpass database, and Visible global -stitched. Refactoring for Python 3 operation, and resolving errors in MODIS processing and RGB MTIF color issue. There -are also breaking changes including modularized PMW algorithms, changed dictionary structures, and standardized platform -names. - -.. toctree:: - :maxdepth: 1 - - v1_1_17 - v1_1_16 - v1_1_15 - v1_1_14 - v1_1_13 diff --git a/docs/source/releases/v1_12_3.rst b/docs/source/releases/v1_12_3.rst index 1cfc85c29..2b20a9e43 100644 --- a/docs/source/releases/v1_12_3.rst +++ b/docs/source/releases/v1_12_3.rst @@ -8,44 +8,42 @@ Version 1.12.3 (2024-05-15) * Formatting Updates + * Allow setting GEOIPS_LOGGING_LEVEL to override default "interactive" * Allow different colors for realtime and admin operational user setups * Bug fixes * Add geolocation variables to geokompsat reader * Add output of radiance to geokompsat reader * Add geolocation variable masking to AHI reader + * Ensure all plugin packages are tested in YAML plugin validation * Ignore missing import in hrit_reader * Fix xarray_to_xarray family * Update ANCILDATDIR to GEOIPS_ANCILDAT * Appropriately pull resolution from LongName - * Fix versions for xarray, rasterio, and cartopy in pyproject.toml + * Check for GEOIPS_DB_URI when using database plugin * Resolve typos in IR_BD and WV colormappers LOG statements - * Update to full path for test_interfaces call in base_test * Resolve typo in netcdf output checker log_with_emphasis call - * Add current miniconda3 install to path - * Check for GEOIPS_DB_URI when using database plugin * Enhancements * Allow passing pyresample kd_tree arguments to interpolators - * Ensure all plugin packages are tested in YAML plugin validation * Allow replacing fields in product_spec_override * Add geolocation resampling to VIIRS reader * Testing Updates - * Use variables[varname] vs getattr(varname) for reader unit tests - * Update config-based AHI backgrounds for AHI geolocation masking update * Minor updates to geokompsat output image title/labels + * Update config-based AHI backgrounds for AHI geolocation masking update + * Use variables[varname] vs getattr(varname) for reader unit tests + * Update to full path for test_interfaces call in base_test * Installation Updates - * Allow passing arguments to pip - * Add general clone_repo command to check_system_requirements + * Fix versions for xarray, rasterio, and cartopy in pyproject.toml + * Add current miniconda3 install to path * Include mamba.sh setup in geoips_conda_init_setup + * Add general clone_repo command to check_system_requirements + * Allow passing arguments to pip * Output version-based requirements.txt immediately after base install. * Require matplotlib<3.9.0 due to matplotlib.cm.get_cmap deprecation * Limit git to <2.45.0 due to bug breaking git lfs clones -* Release updates - - * Allow setting GEOIPS_LOGGING_LEVEL to override default "interactive" * Documentation Updates * Allow including additional sections in plugin package documentation @@ -105,7 +103,6 @@ Ignore missing import in hrit_reader modified: geoips/plugins/modules/readers/utils/hrit_reader.py - Add geolocation variables to geokompsat reader ---------------------------------------------- @@ -150,6 +147,27 @@ Modify xarray_to_xarray family so that it functions correctly. modified: geoips/plugins/modules/procflows/single_source.py +Resolve typo in netcdf output checker log_with_emphasis call +------------------------------------------------------------ + +Fix typo in log_with_emphasis call in netcdf output checker. Was calling for every +line, updated to append all lines into a list, then a single call to log_with_emphasis. + +:: + + modified: geoips/plugins/modules/output_checkers/netcdf.py + +Resolve typos in IR_BD and WV colormappers LOG statements +--------------------------------------------------------- + +Fix typo in IR_BD.py and WV.py colormappers LOG.info statements - previously +had a variable without a corresponding %s. + +:: + + modified: geoips/plugins/modules/colormappers/visir/IR_BD.py + modified: geoips/plugins/modules/colormappers/visir/WV.py + Enhancements ============ @@ -283,30 +301,6 @@ Standardize xarray variable access. modified: tests/unit_tests_long/plugins/modules/readers/test_readers.py -Fix versions for xarray, rasterio, and cartopy in pyproject.toml ----------------------------------------------------------------- - -Require fixed versions for - -* xarray (knmi scat reader bug), -* rasterio (geotiff_standard bug) -* cartopy (test image output gridlines, few pixels difference) - -:: - - modified: pyproject.toml - -Resolve typos in IR_BD and WV colormappers LOG statements ---------------------------------------------------------- - -Fix typo in IR_BD.py and WV.py colormappers LOG.info statements - previously -had a variable without a corresponding %s. - -:: - - modified: geoips/plugins/modules/colormappers/visir/IR_BD.py - modified: geoips/plugins/modules/colormappers/visir/WV.py - Update to full path for test_interfaces call in base_test --------------------------------------------------------- @@ -319,15 +313,21 @@ a pip install of geoips only. modified: tests/integration_tests/base_test.sh -Resolve typo in netcdf output checker log_with_emphasis call ------------------------------------------------------------- +Installation Updates +==================== -Fix typo in log_with_emphasis call in netcdf output checker. Was calling for every -line, updated to append all lines into a list, then a single call to log_with_emphasis. +Fix versions for xarray, rasterio, and cartopy in pyproject.toml +---------------------------------------------------------------- + +Require fixed versions for + +* xarray (knmi scat reader bug), +* rasterio (geotiff_standard bug) +* cartopy (test image output gridlines, few pixels difference) :: - modified: geoips/plugins/modules/output_checkers/netcdf.py + modified: pyproject.toml Add current miniconda3 install to path -------------------------------------- @@ -339,17 +339,6 @@ during conda env calls. modified: setup/geoips_conda_init_setup -Resolve typos in IR_BD and WV colormappers LOG statements ---------------------------------------------------------- - -Fix typo in IR_BD.py and WV.py colormappers LOG.info statements - previously -had a variable without a corresponding %s. - -:: - - modified: geoips/plugins/modules/colormappers/visir/IR_BD.py - modified: geoips/plugins/modules/colormappers/visir/WV.py - Include mamba.sh setup in geoips_conda_init_setup ------------------------------------------------- diff --git a/docs/source/releases/v1_13_1.rst b/docs/source/releases/v1_13_1.rst index 9d1c9036a..8be4b3925 100644 --- a/docs/source/releases/v1_13_1.rst +++ b/docs/source/releases/v1_13_1.rst @@ -3,11 +3,319 @@ | # # # This source code is protected under the license referenced at | # # # https://github.com/NRLMMD-GEOIPS. -Version 1.13.1 (2024-06-01) -***************************** +Version 1.13.1 (2024-07-07) +*************************** -* Update 'update_this_release_note' -* Add release note for v1.13.1 +* Enhancements + + * Add histogram_csv output formatter + * Add oceansat-3 products + * Add get_cmap_data_range to products/dev.py + * Add scipy griddata alpha masking corrections +* Bug Fixes + + * Replace test_data_scat_1.11.2 and 1.11.3 with test_data_scat in unit test + * Update 89pct colormapper data range to match algorithm + * Fixed log double printing +* Installation Updates + + * Add git lfs install to config_geoips + * Add source gitconfigs to base_install + * Freeze git<2.45 and git-lfs<3.4 + * Ensure github downloads also decompress test data + * Allow test_data_github clones with only \*.tgz files + * Freeze netCDF4 < 1.7 due to ABI reader core dump +* Documentation Updates + + * Only required _api and releases docs/source directories + * Require mistune==0.8.4 + * Remove check_old_docs_frozen.py, use git in actions +* Testing Updates + + * Use enviroment variable to control output checker image comparison threshold. + * If tests pass within threshold but not exactly, include exact diff image in logs + * Add bio012024.dat B-deck file + * Remove template_fusion_plugin test calls + * Update test outputs due to minor pixel differences + * Pass clobber=True into atms netcdf test script +* Release Updates + + * Update 'update_this_release_note' + * Add release note for v1.13.1 + * Add release note for v1.13.2, finalize 1.13.1 release + +Enhancements +============ + +Add scipy griddata alpha masking corrections +-------------------------------------------- + +Per recommendations from Nick Zorn MIT LL, implement alpha masking corrections +to help resolve edge of scan issues when using scipy's griddata interpolation + +:: + + modified: geoips/plugins/modules/interpolators/utils/interp_scipy.py + modified: pyproject.toml + +Add get_cmap_data_range to products/dev.py +------------------------------------------ + +To allow pulling ranges separately from algorithm and colormapper plugins, +add an option to geoips/dev/product.py to get the colormapper range. + +This should eventually be added directly to the plugin interfaces, but for now +following the template of get_data_range. + +:: + + modified: geoips/dev/product.py + +Add oceansat-3 products +----------------------- + +Adding sectored/unsectored products, and adding to scat knmi winds. + +:: + + modified: geoips/plugins/modules/readers/scat_knmi_winds_netcdf.py + modified: geoips/plugins/yaml/products/oscat.yaml + +Add histogram_csv output formatter +---------------------------------- + +This was inadvertently not added. + +:: + + new file: geoips/plugins/modules/output_formatters/histogram_csv.py + +Bug Fixes +========= + +Update 89pct colormapper data range to match algorithm +------------------------------------------------------ + +:: + + modified: geoips/plugins/yaml/product_defaults/pmw_89/89pct.yaml + +Replace test_data_scat_1.11.2 and 1.11.3 with test_data_scat in unit test +------------------------------------------------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +:: + + modified: tests/unit_tests/commandline/cli_top_level_tester.py + +Fixed log printing double +------------------------- + +Fixed a bug where the new CLI and the old CLI would both create handlers for logging, +resulting in a double printing for stdout logs, and double lines when running +run_procflow. This was fixed by checking for multiple handlers in logging. + +:: + + modified: geoips/commandline/log_setup.py + +Installation Updates +==================== + +Freeze netCDF4 < 1.7 due to ABI reader core dump +------------------------------------------------ + +*From GEOIPS#513: 2024-06-01, bug fixes* + +ABI Reader core dumps with netCDF 1.7.1. 1.6 still works. + +:: + + modified: pyproject.toml + +Allow test_data_github clones with only \*.tgz files +---------------------------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +Previously required data, outputs, or docs subdir. Allow only a tgz file. + +:: + + modified: setup/check_system_requirements.sh + +Ensure github downloads also decompress test data +------------------------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +Many github repos contain compressed datasets - ensure +data is decompressed after cloning if required. + +:: + + modified: setup/check_system_requirements.sh + +Freeze git<2.45 and git-lfs<3.4 +-------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +Now git and git lfs both break git lfs clones. +Freeze at versions 2.44 and 3.3 until they fix +their bugs. + +:: + + doc/source/starter/installation.rst + +Add git lfs install to config_geoips +------------------------------------ + +*From GEOIPS#513: 2024-06-01, bug fixes* + +:: + + modified: setup/config_geoips + +Add source gitconfigs to base_install +------------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +This includes timeout and buffer size updates that +can help with large clones. + +:: + + modified: setup/check_system_requirements.sh + modified: tests/integration_tests/base_install.sh + +Documentation Updates +===================== + +Remove check_old_docs_frozen.py, use git in actions +--------------------------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +:: + + removed: docs/check_old_docs_frozen.py + modified: .github/workflows/check-old-docs-frozen.yml + +Only required _api and releases docs/source directories +------------------------------------------------------------ + +*From GEOIPS#513: 2024-06-01, bug fixes* + +Make introduction and userguide optional. + +:: + + modified: docs/build_docs.sh + modified: docs/source/_templates/index_PKG.html + +Require mistune==0.8.4 +---------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +Somehow mistune 3.0.2 was installed once, which had errors + +m2r2 = { version = "*", optional = true } +Version 3.0.2 causes error , 0.8.4 works +Unsure why there was such a wide range of version numbers installed. + +:: + + File "lib/python3.10/site-packages/m2r2.py", line 82, in + class RestBlockGrammar(mistune.BlockGrammar): + AttributeError: module 'mistune' has no attribute 'BlockGrammar' + +:: + + modified: pyproject.toml + +Testing Updates +=============== + +Use enviroment variable to control output checker image comparison threshold +---------------------------------------------------------------------------- + +Set OUTPUT_CHECKER_THRESHOLD_IMAGE within base_paths - default to 0.05 if env +variable is not set, else use environemnt variable. Default to base_paths +threshold within the image output checker. + +:: + + modified: geoips/filenames/base_paths.py + modified: geoips/interfaces/module_based/output_checkers.py + modified: geoips/plugins/modules/output_checkers/image.py + +If tests pass within threshold but not exactly, include exact diff image in logs +-------------------------------------------------------------------------------- + +If the test passes the threshold, but images are not identical, include the +exact difference in the log output for reference. This allows tracking what +is actually different, but still allows the test to pass. + +:: + + modified: geoips/interfaces/module_based/output_checkers.py + +Pass clobber=True into atms netcdf test script +---------------------------------------------- + +When running atms netcdf output test script for a second time without setting +clobber=True, the output does not get written so the test fails. Update netcdf +output formatters to take a "clobber" argument, then add that argument to the +output_formatter_kwargs in the atms test script. + +:: + + modified: geoips/plugins/modules/output_formatters/netcdf_geoips.py + modified: geoips/plugins/modules/output_formatters/netcdf_xarray.py + modified: tests/scripts/atms.tc.165H.netcdf_geoips.sh + +Update test outputs due to minor pixel differences +-------------------------------------------------- + +A few coastline pixels difference for ABI and AMSR annotated test outputs. Unsure +what caused this, but updating to get 0 return. + +:: + + modified: tests/outputs/abi.static.Infrared.imagery_annotated/20200918.195020.goes-16.abi.Infrared.goes_east.45p56.noaa.10p0.png + modified: tests/outputs/abi.static.Visible.imagery_annotated/20200918.195020.goes-16.abi.Visible.goes_east.41p12.noaa.10p0.png + modified: tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png + modified: tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png + modified: tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png + modified: tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png + modified: tests/outputs/amsr2.tc.89H-Physical.imagery_annotated/20200518_073601_IO012020_amsr2_gcom-w1_89H-Physical_140kts_100p00_res1p0-cr300.png + modified: tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgInfrared-Gray.png + modified: tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgVisible.png + modified: tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgInfrared-Gray.png + modified: tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgVisible.png + +Remove template_fusion_plugin test calls +---------------------------------------- + +Going to rely on a combination of template_basic_plugin, geoips_plugin_examples, +and data_fusion repos for use cases and examples. Only maintain a single plugin +template repo. + +:: + + modified: tests/integration_tests/full_test.sh + +Add bio012024.dat B-deck file +----------------------------- + +:: + + new file: tests/sectors/tc_bdecks/bio012024.dat Release Process =============== @@ -25,6 +333,19 @@ this release note. modified: docs/source/releases/v1_13_1.rst modified: docs/source/releases/index.rst +Add release note for v1.13.2, finalize 1.13.1 release +----------------------------------------------------- + +*From GEOIPS#513: 2024-07-07, 1.13.1 updates* + +All updates until the next release (v1.13.2) will be included in +this release note. This finalizes the 1.13.1 release. + +:: + + modified: docs/source/releases/v1_13_2.rst + modified: docs/source/releases/index.rst + Update 'update_this_release_note' --------------------------------- diff --git a/docs/source/releases/v1_14_0a0.rst b/docs/source/releases/v1_14_0a0.rst index 4eb69c171..448b68936 100644 --- a/docs/source/releases/v1_14_0a0.rst +++ b/docs/source/releases/v1_14_0a0.rst @@ -1,5 +1,5 @@ .. dropdown:: Distribution Statement - + | # # # This source code is protected under the license referenced at | # # # https://github.com/NRLMMD-GEOIPS. diff --git a/docs/source/releases/version_summaries.yaml b/docs/source/releases/version_summaries.yaml new file mode 100644 index 000000000..417190793 --- /dev/null +++ b/docs/source/releases/version_summaries.yaml @@ -0,0 +1,63 @@ +summaries: +- version: 1.12 + summary: | + Added documentation and enhancements for coding standards, fixed bugs in polar data, log functions, and Dockerfile; + Introduced documentation for PEP8, linters, and plugin registries; Refactored optional imports, plugin registries. Fixed + issues with create_plugin_registries, ATMS platform names, and pytest dependencies. +- version: 1.11 + summary: | + Removed flake8_docstring_only flag from test_all.sh and full_test.sh across the repository. Fixed issues in integration + tests and ensured full flake8 tests applied. +- version: 1.10 + summary: | + Updated interface names throughout the code base, moved YAML sector files to new directories within the geoips/plugins + structure, added support for both the old and new dynamic sector YAML formatting, and renamed plugins to enhance + clarity. Introduced a try/except block in sector_utils/tc_tracks.py to support both old and new dynamic sector YAML + formats. Consolidated both dynamic and static sectors under the unified directory structure in + geoips/plugins/yaml/sectors, and renamed various modules to reflect their functionalities more accurately. +- version: 1.9 + summary: | + Removed GEOIPS BASEDIR references and introduced GEOIPS_TESTDATA_DIR and GEOIPS_DEPENDENCIES_DIR in + filenames/base_paths.py, updated and implemented sphinx-apidoc for Sphinx documentation builds, finalized RST + documentation outline to align with SharePoint documentation, replaced setup.py with pyproject.toml for project setup, + added "cbar_tick_labels" and "cbar_spacing" fields in Rain colormap mpl_color_info, allowed processing when no tropical + cyclones are present in the database, installed geoips[doc, test, lint] dependencies in setup.sh, and configured code + style to ignore flake8 W503 due to conflicts with black. +- version: 1.8 + summary: | + Renamed SSMI/S 89GHz products to 91GHz, updated F-16 SSMI/S scaling, resolved basename error, added geoips_clavrx to + test_full_install, and corrected minor sphinx build warnings. Fixed tests, ensured accurate product naming, and improved + installation and documentation processes. +- version: 1.7 + summary: | + Updated matplotlib version dependency, extended SAR incident angle range, and converted CHANGELOG and documentation + formats to RST. Removed outdated sections, fixed minor documentation issues, and improved documentation organization and + building process. +- version: 1.6 + summary: | + Added resource usage statistics, documentation updates, and code style enforcement checks. Fixed numpy builtin aliases, + product family typos, and output requirements. +- version: 1.5 + summary: | + Updated the Python installation process, added alternate command line arguments functions, created unique storm + directory names for invests, and fixed bugs. Expanded test repository updates and improved metadata handling. +- version: 1.4 + summary: | + Updated documentation and installation requirements, added new test scripts and functionality. Fixed real-time + processing and coverage checks. +- version: 1.3 + summary: | + Replaced instances of 'atcf' with 'tc', removed support for GEOIPSFINAL and GEOIPSTEMP, and consolidated directories + into 'geoips_outdirs.' Added dev/title.py interface, updated template pull request ticket, allowed passing + 'title_copyright,' and added new DATABASESUCCESS and DATABASEFAILURE string checks. +- version: 1.2 + summary: | + Updated bdeck parser, SMOS text winds, test_interfaces.py, attribute names, call signatures; added ABI test scripts, + Software Requirements Specification. Replaced original_source_filename with original_source_filenames; simplified setup, + command line args checking, installation steps. +- version: 1.1 + summary: | + Addition of new features like MODIS reader, stitched product output, sector overpass database, and Visible global + stitched. Refactoring for Python 3 operation, and resolving errors in MODIS processing and RGB MTIF color issue. There + are also breaking changes including modularized PMW algorithms, changed dictionary structures, and standardized platform + names. \ No newline at end of file diff --git a/docs/source/starter/installation.rst b/docs/source/starter/installation.rst index d8dff63a4..bd58472ff 100644 --- a/docs/source/starter/installation.rst +++ b/docs/source/starter/installation.rst @@ -127,6 +127,7 @@ and run integration tests: # Ensure geoips python environment enabled + # Install base GeoIPS package and minimal test datasets. $GEOIPS_PACKAGES_DIR/geoips/tests/integration_tests/base_install.sh # Create the GeoIPS plugin registries diff --git a/docs/source/starter/mac_installation.rst b/docs/source/starter/mac_installation.rst index dbddc307c..0d74ba0d3 100644 --- a/docs/source/starter/mac_installation.rst +++ b/docs/source/starter/mac_installation.rst @@ -155,3 +155,4 @@ look something like below, indicating that none of the tests failed: Total run time: 82 seconds Number data types run: 3 Number data types failed: 0 + diff --git a/docs/update_release_note_index.py b/docs/update_release_note_index.py index a73a86820..b3d6d749c 100644 --- a/docs/update_release_note_index.py +++ b/docs/update_release_note_index.py @@ -1,63 +1,131 @@ #!/bin/env python + +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + """Update release note index with latest version release reference.""" -from sys import argv, exit - -index_filename = argv[1] -add_version = argv[2] - -# Eventually we will likely support actual versions or latest. For now only -# support "latest". -# This may end up being just auto-generating the ENTIRE index.rst, which will -# remove the need for specifying a version at all. For now we are only adding -# the "latest" section to index.rst so the doc build doesn't fail from a non-existent -# reference. In order to auto-generate the entire index.rst, we would have to -# include the high level minor release version summaries somewhere outside index.rst -# (right now the only specification of the high level summaries is in index.rst, -# so we can't entirely auto-generate it). -# This will be revisited after the 2024 workshop. For now this will work. -if add_version != "latest": - print("Actual versions not yet supported - only latest") - exit(1) - # Something like this for adding actual version to index.rst - # Or we may just auto-generate the entire index.rst - # major, minor, bug_fix = add_version.split(".") - # minor_version = f"{major}.{minor}" - # release_note_version = f"v{add_version.replace(".", "_")}" -with open(index_filename, "r") as fobj: - first = True - new_lines = [] - last_line = None - for line in fobj.readlines(): - # Make sure we write all the lines back out - if "----" not in line or not first: - if last_line: - new_lines += [last_line] - # When we get to the first header ----- line, that means we need to - # add the new "minor release" section (at the top). - # If we end up auto-generating the entire index.rst, this will change - # (since we'll just loop through all the release notes, and write out - # the sections one by one. We wouldn't have to keep track of where we - # were at all). - if "----" in line and first: - if add_version == "latest": - first = False - new_lines += ["Latest (version on cutting edge of git)\n"] - new_lines += ["---------------------------------------\n"] - new_lines += ["\n"] - new_lines += [".. toctree::\n"] - new_lines += [" :maxdepth: 1\n"] - new_lines += ["\n"] - new_lines += [" latest\n"] - new_lines += ["\n"] - new_lines += [last_line] - # Something like this to add actual version to index.rst - # elif last_line != f"Version {minor_version}": - # new_lines += ["Version {minor_version}"] - # new_lines += ["-----------------------"] - # new_linst += [""] - last_line = line - - new_lines += [last_line] -# Now write all the lines back out, with the new "latest" section inserted. +from sys import argv +from glob import glob +from pathlib import Path +from os.path import basename, exists + +index_filename = Path(argv[1]) +release_note_path = Path(argv[2]) + +# Note this does NOT include the high level version summaries - those live +# in docs/source/releases/version_summaries.yaml, but are not currently being +# populated in index.rst. When this release note index creation is moved to +# brassy, the version summaries will be pulled from version_summaries.yaml +# to include in the index.rst. + +# Note this will sort release notes properly named as: +# * vX_Y_Z.rst +# * vX_Y_ZaN.rst +# * X.Y.Z.rst +# * X.Y.ZaN.rst + +# Includes "latest" at the top if it exists. + +# No other release note name formatting supported. + +# Include standard top level headers +lines = [".. dropdown:: Distribution Statement\n"] +lines += ["\n"] +lines += [" | This is an auto-generated file.\n"] +lines += [" | Please abide by license packaged with this software.\n"] +lines += ["\n"] +lines += [".. _release_notes:\n"] +lines += ["\n"] +lines += ["Release Notes\n"] +lines += ["*************\n"] +lines += ["\n"] + +# If latest.rst exists, include it in the index.rst +if exists(str(release_note_path / "latest.rst")): + lines += ["Latest (version on cutting edge of git)\n"] + lines += ["---------------------------------------\n"] + lines += ["\n"] + lines += [".. toctree::\n"] + lines += [" :maxdepth: 1\n"] + lines += ["\n"] + lines += [" latest\n"] + lines += ["\n"] + +# Collect all the major, minor, bugfix, alpha version numbers, in a sortable +# manner +versions = {} +# List all the release notes of format v*.rst - ie, don't include index.rst +# or latest.rst, but DO include X_Y_ZaN.rst (alpha releases) +for rst_file in glob(str(release_note_path / "*.rst")): + if basename(rst_file) in ["latest.rst", "index.rst"]: + continue + # print(f"Adding {rst_file}") + # Just leave X.Y.ZaN or X.Y.Z, as appropriate + # This will work with vX_Y_Z.rst, vX_Y_ZaN.rst, X.Y.Z.rst, or X.Y.ZaN.rst + version_num = ( + basename(rst_file).replace("v", "").replace(".rst", "").replace("_", ".") + ) + # print(f"verson {version_num}") + # Default alpha to None and mmb (major/minor/bugfix) to version_num - + # alpha not defined for full releases. + alpha = None + mmb_version_num = version_num + # If there is an "a" in version_num, ie X_Y_ZaN, ensure we capture the + # appropriate alpha version, and remove the aN from the mmb_version_num + if "a" in version_num: + mmb_version_num, alpha = version_num.split("a") + # Now split for major/minor/bugfix from the X_Y_Z version + major, minor, bugfix = mmb_version_num.split(".") + # Zero pad these for sorting + major = f"{int(major):03}" + minor = f"{int(minor):03}" + bugfix = f"{int(bugfix):03}" + + # Only include alpha if it is defined. + if alpha is not None: + alpha = f"{int(alpha):03}" + # Store these in dictionaries so we can loop through in order + if major not in versions: + versions[major] = {} + if minor not in versions[major]: + versions[major][minor] = {} + if bugfix not in versions[major][minor]: + versions[major][minor][bugfix] = [] + # If alphas are defined, store them in a list at the lowest level. + if alpha is not None: + versions[major][minor][bugfix] += [alpha] + +# Now loop through major, minor, bugfix, and alpha, appending sections +# in order as appropriate. +for major in sorted(versions, reverse=True): + for minor in sorted(versions[major], reverse=True): + print(f"Adding {int(major)}.{int(minor)}") + version_header = f"Version {int(major)}.{int(minor)}" + version_underline = "-" * (len(version_header)) + lines += [f"{version_header}\n"] + lines += [f"{version_underline}\n"] + lines += ["\n"] + lines += [".. toctree::\n"] + lines += [" :maxdepth: 1\n"] + lines += ["\n"] + for bugfix in sorted(versions[major][minor], reverse=True): + curr_mmb = f"v{int(major)}_{int(minor)}_{int(bugfix)}" + curr_mmb_period = f"{int(major)}.{int(minor)}.{int(bugfix)}" + # Only add major/minor/bugfix version if it exists - could be + # an alpha release with NO MMB release. + if exists(str(release_note_path / f"{curr_mmb}.rst")): + lines += [f" {curr_mmb}\n"] + elif exists(str(release_note_path / f"{curr_mmb_period}.rst")): + lines += [f" {curr_mmb_period}\n"] + for alpha in sorted(versions[major][minor][bugfix], reverse=True): + if exists(str(release_note_path / f"{curr_mmb}a{int(alpha)}.rst")): + lines += [f" {curr_mmb}a{int(alpha)}\n"] + elif exists( + str(release_note_path / f"{curr_mmb_period}a{int(alpha)}.rst") + ): + lines += [f" {curr_mmb_period}a{int(alpha)}\n"] + lines += ["\n"] +# Now write all the lines we've collected out to index.rst with open(index_filename, "w") as fobj: - fobj.writelines(new_lines) + print(f"Writing {index_filename}") + fobj.writelines(lines) diff --git a/environments/mamba_base_package_list_1.13.1_20240618.yml b/environments/mamba_base_package_list_1.13.1_20240618.yml new file mode 100644 index 000000000..04b66e1c5 --- /dev/null +++ b/environments/mamba_base_package_list_1.13.1_20240618.yml @@ -0,0 +1,211 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +_libgcc_mutex=0.1=conda_forge +_openmp_mutex=4.5=2_gnu +accessible-pygments=0.0.5=pypi_0 +affine=2.4.0=pypi_0 +alabaster=0.7.16=pypi_0 +asciitree=0.3.3=pypi_0 +astroid=3.2.2=pypi_0 +astropy=6.1.1=pypi_0 +astropy-iers-data=0.2024.6.17.0.31.35=pypi_0 +asttokens=2.4.1=pypi_0 +attrs=23.2.0=pypi_0 +babel=2.15.0=pypi_0 +bandit=1.7.9=pypi_0 +beautifulsoup4=4.12.3=pypi_0 +binutils_impl_linux-64=2.40=ha1999f0_7 +black=24.4.2=pypi_0 +bzip2=1.0.8=hd590300_5 +c-ares=1.28.1=hd590300_0 +ca-certificates=2024.6.2=hbcca054_0 +cartopy=0.23.0=pypi_0 +certifi=2024.6.2=pypi_0 +cftime=1.6.4=pypi_0 +charset-normalizer=3.3.2=pypi_0 +click=8.1.7=pypi_0 +click-plugins=1.1.1=pypi_0 +cligj=0.7.2=pypi_0 +cloudpickle=3.0.0=pypi_0 +colorama=0.4.6=pypi_0 +configobj=5.0.8=pypi_0 +contourpy=1.2.1=pypi_0 +coverage=7.5.3=pypi_0 +curl=8.8.0=he654da7_0 +cycler=0.12.1=pypi_0 +dask=2024.6.0=pypi_0 +decorator=5.1.1=pypi_0 +dill=0.3.8=pypi_0 +doc8=1.1.1=pypi_0 +docutils=0.20.1=pypi_0 +donfig=0.8.1.post1=pypi_0 +ephem=4.1.5=pypi_0 +exceptiongroup=1.2.1=pypi_0 +executing=2.0.1=pypi_0 +fasteners=0.19=pypi_0 +flake8=7.1.0=pypi_0 +flake8-docstrings=1.7.0=pypi_0 +flake8-rst=0.8.0=pypi_0 +flake8-rst-docstrings=0.3.0=pypi_0 +fonttools=4.53.0=pypi_0 +fsspec=2024.6.0=pypi_0 +gcc=13.2.0=hc7bed06_10 +gcc_impl_linux-64=13.2.0=h9eb54c0_10 +geoips=1.13.0.post391.dev0=pypi_0 +gettext=0.22.5=h59595ed_2 +gettext-tools=0.22.5=h59595ed_2 +git=2.44.0=pl5321h709897a_0 +git-lfs=3.3.0=ha770c72_0 +gxx=13.2.0=hc7bed06_10 +gxx_impl_linux-64=13.2.0=h2a599c4_10 +h5py=3.11.0=pypi_0 +idna=3.7=pypi_0 +imageio=2.34.1=pypi_0 +imagesize=1.4.1=pypi_0 +importlib-metadata=7.1.0=pypi_0 +iniconfig=2.0.0=pypi_0 +ipython=8.25.0=pypi_0 +isodate=0.6.1=pypi_0 +isort=5.13.2=pypi_0 +jedi=0.19.1=pypi_0 +jinja2=3.1.4=pypi_0 +jsonschema=4.22.0=pypi_0 +jsonschema-specifications=2023.12.1=pypi_0 +kernel-headers_linux-64=2.6.32=he073ed8_17 +keyutils=1.6.1=h166bdaf_0 +kiwisolver=1.4.5=pypi_0 +krb5=1.21.2=h659d440_0 +lazy-loader=0.4=pypi_0 +ld_impl_linux-64=2.40=hf3520f5_7 +libasprintf=0.22.5=h661eb56_2 +libasprintf-devel=0.22.5=h661eb56_2 +libcurl=8.8.0=hca28451_0 +libedit=3.1.20191231=he28a2e2_2 +libev=4.33=hd590300_2 +libexpat=2.6.2=h59595ed_0 +libffi=3.4.2=h7f98852_5 +libgcc-devel_linux-64=13.2.0=hdb50d1a_110 +libgcc-ng=13.2.0=h77fa898_10 +libgettextpo=0.22.5=h59595ed_2 +libgettextpo-devel=0.22.5=h59595ed_2 +libgfortran-ng=13.2.0=h69a702a_10 +libgfortran5=13.2.0=h3d2ce59_10 +libgomp=13.2.0=h77fa898_10 +libiconv=1.17=hd590300_2 +libnghttp2=1.58.0=h47da74e_1 +libnsl=2.0.1=hd590300_0 +libopenblas=0.3.27=pthreads_h413a1c8_0 +libsanitizer=13.2.0=h6ddb7a1_10 +libsqlite=3.46.0=hde9e2c9_0 +libssh2=1.11.0=h0841786_0 +libstdcxx-devel_linux-64=13.2.0=hdb50d1a_110 +libstdcxx-ng=13.2.0=hc0a3c3a_10 +libuuid=2.38.1=h0b41bf4_0 +libxcrypt=4.4.36=hd590300_1 +libzlib=1.3.1=h4ab18f5_1 +locket=1.0.0=pypi_0 +m2r2=0.3.3.post2=pypi_0 +markdown-it-py=3.0.0=pypi_0 +markupsafe=2.1.5=pypi_0 +matplotlib=3.9.0=pypi_0 +matplotlib-inline=0.1.7=pypi_0 +mccabe=0.7.0=pypi_0 +mdurl=0.1.2=pypi_0 +mistune=0.8.4=pypi_0 +mypy-extensions=1.0.0=pypi_0 +ncurses=6.5=h59595ed_0 +netcdf4=1.7.1=pypi_0 +networkx=3.3=pypi_0 +numcodecs=0.12.1=pypi_0 +numexpr=2.8.4=pypi_0 +numpy=1.26.4=pypi_0 +openblas=0.3.27=pthreads_h7a3da1a_0 +openssl=3.3.1=h4ab18f5_0 +packaging=24.1=pypi_0 +pandas=2.2.2=pypi_0 +parso=0.8.4=pypi_0 +partd=1.4.2=pypi_0 +pathspec=0.12.1=pypi_0 +pbr=6.0.0=pypi_0 +pcre2=10.43=hcad00b1_0 +perl=5.32.1=7_hd590300_perl5 +pillow=10.3.0=pypi_0 +pip=24.0=pyhd8ed1ab_0 +pixelmatch=0.3.0=pypi_0 +pluggy=1.5.0=pypi_0 +pooch=1.8.2=pypi_0 +prettier=0.0.7=pypi_0 +prompt-toolkit=3.0.47=pypi_0 +psutil=5.9.8=pypi_0 +pure-eval=0.2.2=pypi_0 +pyaml-env=1.2.1=pypi_0 +pycodestyle=2.12.0=pypi_0 +pydata-sphinx-theme=0.15.3=pypi_0 +pydocstyle=6.3.0=pypi_0 +pyerfa=2.0.1.4=pypi_0 +pyflakes=3.2.0=pypi_0 +pygments=2.18.0=pypi_0 +pyhdf=0.11.4=pypi_0 +pykdtree=1.3.12=pypi_0 +pylint=3.2.3=pypi_0 +pyorbital=1.8.2=pypi_0 +pyparsing=3.1.2=pypi_0 +pyproj=3.6.1=pypi_0 +pypublicdecompwt=2.8.1.5=pypi_0 +pyresample=1.28.3=pypi_0 +pyshp=2.3.1=pypi_0 +pytest=8.2.2=pypi_0 +pytest-cov=5.0.0=pypi_0 +python=3.10.14=hd12c33a_0_cpython +python-dateutil=2.9.0.post0=pypi_0 +pytz=2024.1=pypi_0 +pyyaml=6.0.1=pypi_0 +rasterio=1.3.10=pypi_0 +readline=8.2=h8228510_1 +referencing=0.35.1=pypi_0 +requests=2.32.3=pypi_0 +restructuredtext-lint=1.4.0=pypi_0 +rich=13.7.1=pypi_0 +rpds-py=0.18.1=pypi_0 +satpy=0.49.0=pypi_0 +scikit-image=0.23.2=pypi_0 +scipy=1.13.1=pypi_0 +setuptools=70.0.0=pyhd8ed1ab_0 +shapely=2.0.4=pypi_0 +six=1.16.0=pypi_0 +snowballstemmer=2.2.0=pypi_0 +snuggs=1.4.7=pypi_0 +soupsieve=2.5=pypi_0 +sphinx=7.1.2=pypi_0 +sphinx-design=0.6.0=pypi_0 +sphinxcontrib-applehelp=1.0.8=pypi_0 +sphinxcontrib-autoprogram=0.1.9=pypi_0 +sphinxcontrib-devhelp=1.0.6=pypi_0 +sphinxcontrib-htmlhelp=2.0.5=pypi_0 +sphinxcontrib-jsmath=1.0.1=pypi_0 +sphinxcontrib-qthelp=1.0.7=pypi_0 +sphinxcontrib-serializinghtml=1.1.10=pypi_0 +stack-data=0.6.3=pypi_0 +stevedore=5.2.0=pypi_0 +sysroot_linux-64=2.12=he073ed8_17 +tabulate=0.9.0=pypi_0 +tifffile=2024.5.22=pypi_0 +tk=8.6.13=noxft_h4845f30_101 +tomli=2.0.1=pypi_0 +tomlkit=0.12.5=pypi_0 +toolz=0.12.1=pypi_0 +traitlets=5.14.3=pypi_0 +trollimage=1.23.2=pypi_0 +trollsift=0.5.1=pypi_0 +typing-extensions=4.12.2=pypi_0 +tzdata=2024.1=pypi_0 +urllib3=2.2.2=pypi_0 +wcwidth=0.2.13=pypi_0 +wheel=0.43.0=pyhd8ed1ab_1 +xarray=2024.2.0=pypi_0 +xarray-datatree=0.0.14=pypi_0 +xz=5.2.6=h166bdaf_0 +zarr=2.18.2=pypi_0 +zipp=3.19.2=pypi_0 +zstd=1.5.6=ha6fb4c9_0 diff --git a/environments/mamba_base_package_list_1.13.1_20240704.yml b/environments/mamba_base_package_list_1.13.1_20240704.yml new file mode 100644 index 000000000..c09f93fb9 --- /dev/null +++ b/environments/mamba_base_package_list_1.13.1_20240704.yml @@ -0,0 +1,218 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +_libgcc_mutex=0.1=conda_forge +_openmp_mutex=4.5=2_gnu +accessible-pygments=0.0.5=pypi_0 +affine=2.4.0=pypi_0 +alabaster=0.7.16=pypi_0 +alphashape=1.3.1=pypi_0 +asciitree=0.3.3=pypi_0 +astroid=3.2.2=pypi_0 +astropy=6.1.1=pypi_0 +astropy-iers-data=0.2024.7.1.0.34.3=pypi_0 +asttokens=2.4.1=pypi_0 +attrs=23.2.0=pypi_0 +babel=2.15.0=pypi_0 +bandit=1.7.9=pypi_0 +beautifulsoup4=4.12.3=pypi_0 +binutils_impl_linux-64=2.40=ha1999f0_7 +black=24.4.2=pypi_0 +bzip2=1.0.8=hd590300_5 +c-ares=1.28.1=hd590300_0 +ca-certificates=2024.7.4=hbcca054_0 +cartopy=0.23.0=pypi_0 +certifi=2024.7.4=pypi_0 +cftime=1.6.4=pypi_0 +charset-normalizer=3.3.2=pypi_0 +click=8.1.7=pypi_0 +click-log=0.4.0=pypi_0 +click-plugins=1.1.1=pypi_0 +cligj=0.7.2=pypi_0 +cloudpickle=3.0.0=pypi_0 +colorama=0.4.6=pypi_0 +configobj=5.0.8=pypi_0 +contourpy=1.2.1=pypi_0 +coverage=7.5.4=pypi_0 +curl=8.8.0=he654da7_1 +cycler=0.12.1=pypi_0 +dask=2024.6.2=pypi_0 +decorator=5.1.1=pypi_0 +dill=0.3.8=pypi_0 +doc8=1.1.1=pypi_0 +docutils=0.20.1=pypi_0 +donfig=0.8.1.post1=pypi_0 +ephem=4.1.5=pypi_0 +exceptiongroup=1.2.1=pypi_0 +executing=2.0.1=pypi_0 +fasteners=0.19=pypi_0 +flake8=7.1.0=pypi_0 +flake8-docstrings=1.7.0=pypi_0 +flake8-rst=0.8.0=pypi_0 +flake8-rst-docstrings=0.3.0=pypi_0 +fonttools=4.53.0=pypi_0 +fsspec=2024.6.1=pypi_0 +gcc=14.1.0=h6f9ffa1_0 +gcc_impl_linux-64=14.1.0=h3c94d91_0 +geoips=1.13.1.post1.dev0=pypi_0 +gettext=0.22.5=h59595ed_2 +gettext-tools=0.22.5=h59595ed_2 +git=2.44.0=pl5321h709897a_0 +git-lfs=3.3.0=ha770c72_0 +gxx=14.1.0=h6f9ffa1_0 +gxx_impl_linux-64=14.1.0=h2879b86_0 +h5py=3.11.0=pypi_0 +idna=3.7=pypi_0 +imageio=2.34.2=pypi_0 +imagesize=1.4.1=pypi_0 +importlib-metadata=8.0.0=pypi_0 +iniconfig=2.0.0=pypi_0 +ipython=8.26.0=pypi_0 +isodate=0.6.1=pypi_0 +isort=5.13.2=pypi_0 +jedi=0.19.1=pypi_0 +jinja2=3.1.4=pypi_0 +jsonschema=4.22.0=pypi_0 +jsonschema-specifications=2023.12.1=pypi_0 +kernel-headers_linux-64=2.6.32=he073ed8_17 +keyutils=1.6.1=h166bdaf_0 +kiwisolver=1.4.5=pypi_0 +krb5=1.21.3=h659f571_0 +lazy-loader=0.4=pypi_0 +ld_impl_linux-64=2.40=hf3520f5_7 +libasprintf=0.22.5=h661eb56_2 +libasprintf-devel=0.22.5=h661eb56_2 +libcurl=8.8.0=hca28451_1 +libedit=3.1.20191231=he28a2e2_2 +libev=4.33=hd590300_2 +libexpat=2.6.2=h59595ed_0 +libffi=3.4.2=h7f98852_5 +libgcc-devel_linux-64=14.1.0=h5d3d1c9_100 +libgcc-ng=14.1.0=h77fa898_0 +libgettextpo=0.22.5=h59595ed_2 +libgettextpo-devel=0.22.5=h59595ed_2 +libgfortran-ng=14.1.0=h69a702a_0 +libgfortran5=14.1.0=hc5f4f2c_0 +libgomp=14.1.0=h77fa898_0 +libiconv=1.17=hd590300_2 +libnghttp2=1.58.0=h47da74e_1 +libnsl=2.0.1=hd590300_0 +libopenblas=0.3.27=pthreads_h413a1c8_0 +libsanitizer=14.1.0=hcba0ae0_0 +libsqlite=3.46.0=hde9e2c9_0 +libssh2=1.11.0=h0841786_0 +libstdcxx-devel_linux-64=14.1.0=h5d3d1c9_100 +libstdcxx-ng=14.1.0=hc0a3c3a_0 +libuuid=2.38.1=h0b41bf4_0 +libxcrypt=4.4.36=hd590300_1 +libzlib=1.3.1=h4ab18f5_1 +locket=1.0.0=pypi_0 +m2r2=0.3.3.post2=pypi_0 +markdown-it-py=3.0.0=pypi_0 +markupsafe=2.1.5=pypi_0 +matplotlib=3.9.1=pypi_0 +matplotlib-inline=0.1.7=pypi_0 +mccabe=0.7.0=pypi_0 +mdurl=0.1.2=pypi_0 +mistune=0.8.4=pypi_0 +mypy-extensions=1.0.0=pypi_0 +ncurses=6.5=h59595ed_0 +netcdf4=1.6.5=pypi_0 +networkx=3.3=pypi_0 +numcodecs=0.12.1=pypi_0 +numexpr=2.8.4=pypi_0 +numpy=1.26.4=pypi_0 +openblas=0.3.27=pthreads_h7a3da1a_0 +openssl=3.3.1=h4ab18f5_1 +packaging=24.1=pypi_0 +pandas=2.2.2=pypi_0 +parso=0.8.4=pypi_0 +partd=1.4.2=pypi_0 +pathspec=0.12.1=pypi_0 +pbr=6.0.0=pypi_0 +pcre2=10.43=hcad00b1_0 +perl=5.32.1=7_hd590300_perl5 +pexpect=4.9.0=pypi_0 +pillow=10.4.0=pypi_0 +pip=24.0=pyhd8ed1ab_0 +pixelmatch=0.3.0=pypi_0 +platformdirs=4.2.2=pypi_0 +pluggy=1.5.0=pypi_0 +pooch=1.8.2=pypi_0 +prettier=0.0.7=pypi_0 +prompt-toolkit=3.0.47=pypi_0 +psutil=6.0.0=pypi_0 +ptyprocess=0.7.0=pypi_0 +pure-eval=0.2.2=pypi_0 +pyaml-env=1.2.1=pypi_0 +pycodestyle=2.12.0=pypi_0 +pydata-sphinx-theme=0.15.4=pypi_0 +pydocstyle=6.3.0=pypi_0 +pyerfa=2.0.1.4=pypi_0 +pyflakes=3.2.0=pypi_0 +pygments=2.18.0=pypi_0 +pyhdf=0.11.4=pypi_0 +pykdtree=1.3.12=pypi_0 +pylint=3.2.5=pypi_0 +pyorbital=1.8.3=pypi_0 +pyparsing=3.1.2=pypi_0 +pyproj=3.6.1=pypi_0 +pypublicdecompwt=2.8.1.5=pypi_0 +pyresample=1.28.4=pypi_0 +pyshp=2.3.1=pypi_0 +pytest=8.2.2=pypi_0 +pytest-cov=5.0.0=pypi_0 +python=3.10.14=hd12c33a_0_cpython +python-dateutil=2.9.0.post0=pypi_0 +pytz=2024.1=pypi_0 +pyyaml=6.0.1=pypi_0 +rasterio=1.3.10=pypi_0 +readline=8.2=h8228510_1 +referencing=0.35.1=pypi_0 +requests=2.32.3=pypi_0 +restructuredtext-lint=1.4.0=pypi_0 +rich=13.7.1=pypi_0 +rpds-py=0.18.1=pypi_0 +rtree=1.2.0=pypi_0 +satpy=0.49.0=pypi_0 +scikit-image=0.24.0=pypi_0 +scipy=1.14.0=pypi_0 +setuptools=70.1.1=pyhd8ed1ab_0 +shapely=2.0.4=pypi_0 +six=1.16.0=pypi_0 +snowballstemmer=2.2.0=pypi_0 +snuggs=1.4.7=pypi_0 +soupsieve=2.5=pypi_0 +sphinx=7.1.2=pypi_0 +sphinx-design=0.6.0=pypi_0 +sphinxcontrib-applehelp=1.0.8=pypi_0 +sphinxcontrib-autoprogram=0.1.9=pypi_0 +sphinxcontrib-devhelp=1.0.6=pypi_0 +sphinxcontrib-htmlhelp=2.0.5=pypi_0 +sphinxcontrib-jsmath=1.0.1=pypi_0 +sphinxcontrib-qthelp=1.0.7=pypi_0 +sphinxcontrib-serializinghtml=1.1.10=pypi_0 +stack-data=0.6.3=pypi_0 +stevedore=5.2.0=pypi_0 +sysroot_linux-64=2.12=he073ed8_17 +tabulate=0.9.0=pypi_0 +tifffile=2024.7.2=pypi_0 +tk=8.6.13=noxft_h4845f30_101 +tomli=2.0.1=pypi_0 +tomlkit=0.12.5=pypi_0 +toolz=0.12.1=pypi_0 +traitlets=5.14.3=pypi_0 +trimesh=4.4.1=pypi_0 +trollimage=1.24.0=pypi_0 +trollsift=0.5.1=pypi_0 +typing-extensions=4.12.2=pypi_0 +tzdata=2024.1=pypi_0 +urllib3=2.2.2=pypi_0 +wcwidth=0.2.13=pypi_0 +wheel=0.43.0=pyhd8ed1ab_1 +xarray=2024.2.0=pypi_0 +xarray-datatree=0.0.14=pypi_0 +xz=5.2.6=h166bdaf_0 +zarr=2.18.2=pypi_0 +zipp=3.19.2=pypi_0 +zstd=1.5.6=ha6fb4c9_0 diff --git a/environments/mamba_base_package_list_1.14.0_20240915.yml b/environments/mamba_base_package_list_1.14.0_20240915.yml new file mode 100644 index 000000000..d04903d48 --- /dev/null +++ b/environments/mamba_base_package_list_1.14.0_20240915.yml @@ -0,0 +1,232 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +_libgcc_mutex=0.1=conda_forge +_openmp_mutex=4.5=2_gnu +_sysroot_linux-64_curr_repodata_hack=3=h69a702a_16 +accessible-pygments=0.0.5=pypi_0 +affine=2.4.0=pypi_0 +alabaster=0.7.16=pypi_0 +alphashape=1.3.1=pypi_0 +annotated-types=0.7.0=pypi_0 +asciitree=0.3.3=pypi_0 +astroid=3.2.4=pypi_0 +astropy=6.1.3=pypi_0 +astropy-iers-data=0.2024.9.12.13.29.57=pypi_0 +asttokens=2.4.1=pypi_0 +attrs=24.2.0=pypi_0 +babel=2.16.0=pypi_0 +bandit=1.7.9=pypi_0 +beautifulsoup4=4.12.3=pypi_0 +binutils_impl_linux-64=2.40=ha1999f0_7 +black=24.8.0=pypi_0 +brassy=0.0.2.post2=pypi_0 +bzip2=1.0.8=h4bc722e_7 +c-ares=1.32.3=h4bc722e_0 +ca-certificates=2024.8.30=hbcca054_0 +cartopy=0.23.0=pypi_0 +certifi=2024.8.30=pypi_0 +cffi=1.17.1=pypi_0 +cftime=1.6.4=pypi_0 +charset-normalizer=3.3.2=pypi_0 +click=8.1.7=pypi_0 +click-log=0.4.0=pypi_0 +click-plugins=1.1.1=pypi_0 +cligj=0.7.2=pypi_0 +cloudpickle=3.0.0=pypi_0 +colorama=0.4.6=pypi_0 +configobj=5.0.8=pypi_0 +contourpy=1.3.0=pypi_0 +coverage=7.6.1=pypi_0 +curl=8.10.0=hbbe4b11_0 +cycler=0.12.1=pypi_0 +dask=2024.9.0=pypi_0 +decorator=5.1.1=pypi_0 +dill=0.3.8=pypi_0 +doc8=1.1.2=pypi_0 +docutils=0.20.1=pypi_0 +donfig=0.8.1.post1=pypi_0 +ephem=4.1.5=pypi_0 +exceptiongroup=1.2.2=pypi_0 +executing=2.1.0=pypi_0 +fasteners=0.19=pypi_0 +flake8=7.1.1=pypi_0 +flake8-docstrings=1.7.0=pypi_0 +flake8-rst=0.8.0=pypi_0 +flake8-rst-docstrings=0.3.0=pypi_0 +fonttools=4.53.1=pypi_0 +fsspec=2024.9.0=pypi_0 +gcc=14.1.0=h6f9ffa1_1 +gcc_impl_linux-64=14.1.0=h3c94d91_1 +geoips=1.14.0=pypi_0 +gettext=0.22.5=he02047a_3 +gettext-tools=0.22.5=he02047a_3 +git=2.44.0=pl5321h709897a_0 +git-lfs=3.3.0=ha770c72_0 +gxx=14.1.0=h6f9ffa1_1 +gxx_impl_linux-64=14.1.0=h8d00ecb_1 +h5py=3.11.0=pypi_0 +idna=3.9=pypi_0 +imageio=2.35.1=pypi_0 +imagesize=1.4.1=pypi_0 +importlib-metadata=8.5.0=pypi_0 +iniconfig=2.0.0=pypi_0 +ipython=8.27.0=pypi_0 +isodate=0.6.1=pypi_0 +isort=5.13.2=pypi_0 +jedi=0.19.1=pypi_0 +jinja2=3.1.4=pypi_0 +jsonschema=4.23.0=pypi_0 +jsonschema-specifications=2023.12.1=pypi_0 +kernel-headers_linux-64=3.10.0=h4a8ded7_16 +keyutils=1.6.1=h166bdaf_0 +kiwisolver=1.4.7=pypi_0 +krb5=1.21.3=h659f571_0 +lazy-loader=0.4=pypi_0 +ld_impl_linux-64=2.40=hf3520f5_7 +libasprintf=0.22.5=he8f35ee_3 +libasprintf-devel=0.22.5=he8f35ee_3 +libcurl=8.10.0=hbbe4b11_0 +libedit=3.1.20191231=he28a2e2_2 +libev=4.33=hd590300_2 +libexpat=2.6.3=h5888daf_0 +libffi=3.4.2=h7f98852_5 +libgcc=14.1.0=h77fa898_1 +libgcc-devel_linux-64=14.1.0=h5d3d1c9_101 +libgcc-ng=14.1.0=h69a702a_1 +libgettextpo=0.22.5=he02047a_3 +libgettextpo-devel=0.22.5=he02047a_3 +libgfortran=14.1.0=h69a702a_1 +libgfortran-ng=14.1.0=h69a702a_1 +libgfortran5=14.1.0=hc5f4f2c_1 +libgomp=14.1.0=h77fa898_1 +libiconv=1.17=hd590300_2 +libnghttp2=1.58.0=h47da74e_1 +libnsl=2.0.1=hd590300_0 +libopenblas=0.3.28=pthreads_h94d23a6_0 +libsanitizer=14.1.0=hcba0ae0_1 +libsqlite=3.46.1=hadc24fc_0 +libssh2=1.11.0=h0841786_0 +libstdcxx=14.1.0=hc0a3c3a_1 +libstdcxx-devel_linux-64=14.1.0=h5d3d1c9_101 +libstdcxx-ng=14.1.0=h4852527_1 +libuuid=2.38.1=h0b41bf4_0 +libxcrypt=4.4.36=hd590300_1 +libzlib=1.3.1=h4ab18f5_1 +locket=1.0.0=pypi_0 +m2r2=0.3.3.post2=pypi_0 +markdown-it-py=3.0.0=pypi_0 +markupsafe=2.1.5=pypi_0 +matplotlib=3.9.2=pypi_0 +matplotlib-inline=0.1.7=pypi_0 +mccabe=0.7.0=pypi_0 +mdurl=0.1.2=pypi_0 +mistune=0.8.4=pypi_0 +morecantile=5.4.2=pypi_0 +mypy-extensions=1.0.0=pypi_0 +ncurses=6.5=he02047a_1 +netcdf4=1.7.1.post2=pypi_0 +networkx=3.3=pypi_0 +numcodecs=0.13.0=pypi_0 +numexpr=2.8.4=pypi_0 +numpy=1.26.4=pypi_0 +openblas=0.3.28=pthreads_hbcdf1e8_0 +openssl=3.3.2=hb9d3cd8_0 +packaging=24.1=pypi_0 +pandas=2.2.2=pypi_0 +parso=0.8.4=pypi_0 +partd=1.4.2=pypi_0 +pathspec=0.12.1=pypi_0 +pbr=6.1.0=pypi_0 +pcre2=10.43=hcad00b1_0 +perl=5.32.1=7_hd590300_perl5 +pillow=10.4.0=pypi_0 +pinkrst=0.0.1=pypi_0 +pip=24.2=pyh8b19718_1 +pixelmatch=0.3.0=pypi_0 +platformdirs=4.3.3=pypi_0 +pluggy=1.5.0=pypi_0 +pooch=1.8.2=pypi_0 +prettier=0.0.7=pypi_0 +prompt-toolkit=3.0.47=pypi_0 +psutil=6.0.0=pypi_0 +pure-eval=0.2.3=pypi_0 +pyaml-env=1.2.1=pypi_0 +pycodestyle=2.12.1=pypi_0 +pycparser=2.22=pypi_0 +pydantic=2.9.1=pypi_0 +pydantic-core=2.23.3=pypi_0 +pydata-sphinx-theme=0.15.4=pypi_0 +pydocstyle=6.3.0=pypi_0 +pyerfa=2.0.1.4=pypi_0 +pyflakes=3.2.0=pypi_0 +pygit2=1.15.1=pypi_0 +pygments=2.18.0=pypi_0 +pyhdf=0.11.4=pypi_0 +pykdtree=1.3.13=pypi_0 +pylint=3.2.7=pypi_0 +pyorbital=1.8.3=pypi_0 +pyparsing=3.1.4=pypi_0 +pyproj=3.6.1=pypi_0 +pypublicdecompwt=2.8.1.6=pypi_0 +pyresample=1.30.0=pypi_0 +pyshp=2.3.1=pypi_0 +pytest=8.3.3=pypi_0 +pytest-cov=5.0.0=pypi_0 +python=3.10.14=hd12c33a_0_cpython +python-dateutil=2.9.0.post0=pypi_0 +pytz=2024.2=pypi_0 +pyyaml=6.0.2=pypi_0 +rasterio=1.3.11=pypi_0 +readline=8.2=h8228510_1 +referencing=0.35.1=pypi_0 +requests=2.32.3=pypi_0 +restructuredtext-lint=1.4.0=pypi_0 +rich=13.8.1=pypi_0 +rich-argparse=1.5.2=pypi_0 +rio-cogeo=5.3.4=pypi_0 +rpds-py=0.20.0=pypi_0 +rtree=1.3.0=pypi_0 +satpy=0.51.0=pypi_0 +scikit-image=0.24.0=pypi_0 +scipy=1.14.1=pypi_0 +setuptools=73.0.1=pyhd8ed1ab_0 +shapely=2.0.6=pypi_0 +six=1.16.0=pypi_0 +snowballstemmer=2.2.0=pypi_0 +snuggs=1.4.7=pypi_0 +soupsieve=2.6=pypi_0 +sphinx=7.1.2=pypi_0 +sphinx-argparse=0.5.2=pypi_0 +sphinx-design=0.6.1=pypi_0 +sphinxcontrib-applehelp=2.0.0=pypi_0 +sphinxcontrib-autoprogram=0.1.9=pypi_0 +sphinxcontrib-devhelp=2.0.0=pypi_0 +sphinxcontrib-htmlhelp=2.1.0=pypi_0 +sphinxcontrib-jsmath=1.0.1=pypi_0 +sphinxcontrib-qthelp=2.0.0=pypi_0 +sphinxcontrib-serializinghtml=2.0.0=pypi_0 +stack-data=0.6.3=pypi_0 +stevedore=5.3.0=pypi_0 +sysroot_linux-64=2.17=h4a8ded7_16 +tabulate=0.9.0=pypi_0 +tifffile=2024.8.30=pypi_0 +tk=8.6.13=noxft_h4845f30_101 +tomli=2.0.1=pypi_0 +tomlkit=0.13.2=pypi_0 +toolz=0.12.1=pypi_0 +traitlets=5.14.3=pypi_0 +trimesh=4.4.9=pypi_0 +trollimage=1.25.0=pypi_0 +trollsift=0.5.1=pypi_0 +typing-extensions=4.12.2=pypi_0 +tzdata=2024.1=pypi_0 +urllib3=2.2.3=pypi_0 +wcwidth=0.2.13=pypi_0 +wheel=0.44.0=pyhd8ed1ab_0 +xarray=2024.2.0=pypi_0 +xarray-datatree=0.0.14=pypi_0 +xz=5.2.6=h166bdaf_0 +zarr=2.18.3=pypi_0 +zipp=3.20.2=pypi_0 +zstd=1.5.6=ha6fb4c9_0 diff --git a/environments/mamba_full_package_list_1.14.0_20240916.yml b/environments/mamba_full_package_list_1.14.0_20240916.yml new file mode 100644 index 000000000..e1d59e23b --- /dev/null +++ b/environments/mamba_full_package_list_1.14.0_20240916.yml @@ -0,0 +1,239 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +_libgcc_mutex=0.1=conda_forge +_openmp_mutex=4.5=2_gnu +_sysroot_linux-64_curr_repodata_hack=3=h69a702a_16 +accessible-pygments=0.0.5=pypi_0 +affine=2.4.0=pypi_0 +akima86=0.1.1=pypi_0 +alabaster=0.7.16=pypi_0 +alphashape=1.3.1=pypi_0 +annotated-types=0.7.0=pypi_0 +archer=1.0=pypi_0 +asciitree=0.3.3=pypi_0 +astroid=3.2.4=pypi_0 +astropy=6.1.3=pypi_0 +astropy-iers-data=0.2024.9.12.13.29.57=pypi_0 +asttokens=2.4.1=pypi_0 +attrs=24.2.0=pypi_0 +babel=2.16.0=pypi_0 +bandit=1.7.9=pypi_0 +beautifulsoup4=4.12.3=pypi_0 +binutils_impl_linux-64=2.40=ha1999f0_7 +black=24.8.0=pypi_0 +brassy=0.0.2.post2=pypi_0 +bzip2=1.0.8=h4bc722e_7 +c-ares=1.32.3=h4bc722e_0 +ca-certificates=2024.8.30=hbcca054_0 +cartopy=0.23.0=pypi_0 +certifi=2024.8.30=pypi_0 +cffi=1.17.1=pypi_0 +cftime=1.6.4=pypi_0 +charset-normalizer=3.3.2=pypi_0 +click=8.1.7=pypi_0 +click-log=0.4.0=pypi_0 +click-plugins=1.1.1=pypi_0 +cligj=0.7.2=pypi_0 +cloudpickle=3.0.0=pypi_0 +colorama=0.4.6=pypi_0 +configobj=5.0.8=pypi_0 +contourpy=1.3.0=pypi_0 +coverage=7.6.1=pypi_0 +curl=8.10.0=hbbe4b11_0 +cycler=0.12.1=pypi_0 +dask=2024.9.0=pypi_0 +data-fusion=1.14.0=pypi_0 +decorator=5.1.1=pypi_0 +dill=0.3.8=pypi_0 +doc8=1.1.2=pypi_0 +docutils=0.20.1=pypi_0 +donfig=0.8.1.post1=pypi_0 +ephem=4.1.5=pypi_0 +exceptiongroup=1.2.2=pypi_0 +executing=2.1.0=pypi_0 +fasteners=0.19=pypi_0 +flake8=7.1.1=pypi_0 +flake8-docstrings=1.7.0=pypi_0 +flake8-rst=0.8.0=pypi_0 +flake8-rst-docstrings=0.3.0=pypi_0 +fonttools=4.53.1=pypi_0 +fsspec=2024.9.0=pypi_0 +gcc=14.1.0=h6f9ffa1_1 +gcc_impl_linux-64=14.1.0=h3c94d91_1 +geoips=1.14.0=pypi_0 +geoips-clavrx=1.13.0.post1.dev0=pypi_0 +geoips-plugin-example=1.13.0.post1.dev0=pypi_0 +gettext=0.22.5=he02047a_3 +gettext-tools=0.22.5=he02047a_3 +git=2.44.0=pl5321h709897a_0 +git-lfs=3.3.0=ha770c72_0 +gxx=14.1.0=h6f9ffa1_1 +gxx_impl_linux-64=14.1.0=h8d00ecb_1 +h5py=3.11.0=pypi_0 +idna=3.9=pypi_0 +imageio=2.35.1=pypi_0 +imagesize=1.4.1=pypi_0 +importlib-metadata=8.5.0=pypi_0 +iniconfig=2.0.0=pypi_0 +ipython=8.27.0=pypi_0 +isodate=0.6.1=pypi_0 +isort=5.13.2=pypi_0 +jedi=0.19.1=pypi_0 +jinja2=3.1.4=pypi_0 +jsonschema=4.23.0=pypi_0 +jsonschema-specifications=2023.12.1=pypi_0 +kernel-headers_linux-64=3.10.0=h4a8ded7_16 +keyutils=1.6.1=h166bdaf_0 +kiwisolver=1.4.7=pypi_0 +krb5=1.21.3=h659f571_0 +lazy-loader=0.4=pypi_0 +ld_impl_linux-64=2.40=hf3520f5_7 +libasprintf=0.22.5=he8f35ee_3 +libasprintf-devel=0.22.5=he8f35ee_3 +libcurl=8.10.0=hbbe4b11_0 +libedit=3.1.20191231=he28a2e2_2 +libev=4.33=hd590300_2 +libexpat=2.6.3=h5888daf_0 +libffi=3.4.2=h7f98852_5 +libgcc=14.1.0=h77fa898_1 +libgcc-devel_linux-64=14.1.0=h5d3d1c9_101 +libgcc-ng=14.1.0=h69a702a_1 +libgettextpo=0.22.5=he02047a_3 +libgettextpo-devel=0.22.5=he02047a_3 +libgfortran=14.1.0=h69a702a_1 +libgfortran-ng=14.1.0=h69a702a_1 +libgfortran5=14.1.0=hc5f4f2c_1 +libgomp=14.1.0=h77fa898_1 +libiconv=1.17=hd590300_2 +libnghttp2=1.58.0=h47da74e_1 +libnsl=2.0.1=hd590300_0 +libopenblas=0.3.28=pthreads_h94d23a6_0 +libsanitizer=14.1.0=hcba0ae0_1 +libsqlite=3.46.1=hadc24fc_0 +libssh2=1.11.0=h0841786_0 +libstdcxx=14.1.0=hc0a3c3a_1 +libstdcxx-devel_linux-64=14.1.0=h5d3d1c9_101 +libstdcxx-ng=14.1.0=h4852527_1 +libuuid=2.38.1=h0b41bf4_0 +libxcrypt=4.4.36=hd590300_1 +libzlib=1.3.1=h4ab18f5_1 +locket=1.0.0=pypi_0 +m2r2=0.3.3.post2=pypi_0 +markdown-it-py=3.0.0=pypi_0 +markupsafe=2.1.5=pypi_0 +matplotlib=3.9.2=pypi_0 +matplotlib-inline=0.1.7=pypi_0 +mccabe=0.7.0=pypi_0 +mdurl=0.1.2=pypi_0 +mistune=0.8.4=pypi_0 +morecantile=5.4.2=pypi_0 +my-package=1.13.1=pypi_0 +mypy-extensions=1.0.0=pypi_0 +ncurses=6.5=he02047a_1 +netcdf4=1.7.1.post2=pypi_0 +networkx=3.3=pypi_0 +numcodecs=0.13.0=pypi_0 +numexpr=2.8.4=pypi_0 +numpy=1.26.4=pypi_0 +openblas=0.3.28=pthreads_hbcdf1e8_0 +openssl=3.3.2=hb9d3cd8_0 +packaging=24.1=pypi_0 +pandas=2.2.2=pypi_0 +parso=0.8.4=pypi_0 +partd=1.4.2=pypi_0 +pathspec=0.12.1=pypi_0 +pbr=6.1.0=pypi_0 +pcre2=10.43=hcad00b1_0 +perl=5.32.1=7_hd590300_perl5 +pillow=10.4.0=pypi_0 +pinkrst=0.0.1=pypi_0 +pip=24.2=pyh8b19718_1 +pixelmatch=0.3.0=pypi_0 +platformdirs=4.3.3=pypi_0 +pluggy=1.5.0=pypi_0 +pooch=1.8.2=pypi_0 +prettier=0.0.7=pypi_0 +prompt-toolkit=3.0.47=pypi_0 +psutil=6.0.0=pypi_0 +pure-eval=0.2.3=pypi_0 +pyaml-env=1.2.1=pypi_0 +pycodestyle=2.12.1=pypi_0 +pycparser=2.22=pypi_0 +pydantic=2.9.1=pypi_0 +pydantic-core=2.23.3=pypi_0 +pydata-sphinx-theme=0.15.4=pypi_0 +pydocstyle=6.3.0=pypi_0 +pyerfa=2.0.1.4=pypi_0 +pyflakes=3.2.0=pypi_0 +pygit2=1.15.1=pypi_0 +pygments=2.18.0=pypi_0 +pyhdf=0.11.4=pypi_0 +pykdtree=1.3.13=pypi_0 +pylint=3.2.7=pypi_0 +pyorbital=1.8.3=pypi_0 +pyparsing=3.1.4=pypi_0 +pyproj=3.6.1=pypi_0 +pypublicdecompwt=2.8.1.6=pypi_0 +pyresample=1.30.0=pypi_0 +pyshp=2.3.1=pypi_0 +pytest=8.3.3=pypi_0 +pytest-cov=5.0.0=pypi_0 +python=3.10.14=hd12c33a_0_cpython +python-dateutil=2.9.0.post0=pypi_0 +pytz=2024.2=pypi_0 +pyyaml=6.0.2=pypi_0 +rasterio=1.3.11=pypi_0 +readline=8.2=h8228510_1 +recenter-tc=1.14.0a0.post3.dev0=pypi_0 +referencing=0.35.1=pypi_0 +requests=2.32.3=pypi_0 +restructuredtext-lint=1.4.0=pypi_0 +rich=13.8.1=pypi_0 +rich-argparse=1.5.2=pypi_0 +rio-cogeo=5.3.4=pypi_0 +rpds-py=0.20.0=pypi_0 +rtree=1.3.0=pypi_0 +satpy=0.51.0=pypi_0 +scikit-image=0.24.0=pypi_0 +scipy=1.14.1=pypi_0 +setuptools=69.5.1=pypi_0 +shapely=2.0.6=pypi_0 +six=1.16.0=pypi_0 +snowballstemmer=2.2.0=pypi_0 +snuggs=1.4.7=pypi_0 +soupsieve=2.6=pypi_0 +sphinx=7.1.2=pypi_0 +sphinx-argparse=0.5.2=pypi_0 +sphinx-design=0.6.1=pypi_0 +sphinxcontrib-applehelp=2.0.0=pypi_0 +sphinxcontrib-autoprogram=0.1.9=pypi_0 +sphinxcontrib-devhelp=2.0.0=pypi_0 +sphinxcontrib-htmlhelp=2.1.0=pypi_0 +sphinxcontrib-jsmath=1.0.1=pypi_0 +sphinxcontrib-qthelp=2.0.0=pypi_0 +sphinxcontrib-serializinghtml=2.0.0=pypi_0 +stack-data=0.6.3=pypi_0 +stevedore=5.3.0=pypi_0 +sysroot_linux-64=2.17=h4a8ded7_16 +tabulate=0.9.0=pypi_0 +tifffile=2024.8.30=pypi_0 +tk=8.6.13=noxft_h4845f30_101 +tomli=2.0.1=pypi_0 +tomlkit=0.13.2=pypi_0 +toolz=0.12.1=pypi_0 +traitlets=5.14.3=pypi_0 +trimesh=4.4.9=pypi_0 +trollimage=1.25.0=pypi_0 +trollsift=0.5.1=pypi_0 +typing-extensions=4.12.2=pypi_0 +tzdata=2024.1=pypi_0 +urllib3=2.2.3=pypi_0 +wcwidth=0.2.13=pypi_0 +wheel=0.44.0=pyhd8ed1ab_0 +xarray=2024.2.0=pypi_0 +xarray-datatree=0.0.14=pypi_0 +xz=5.2.6=h166bdaf_0 +zarr=2.18.3=pypi_0 +zipp=3.20.2=pypi_0 +zstd=1.5.6=ha6fb4c9_0 diff --git a/environments/pip_base_requirements_1.13.1_20240618.txt b/environments/pip_base_requirements_1.13.1_20240618.txt new file mode 100644 index 000000000..f465a8c2f --- /dev/null +++ b/environments/pip_base_requirements_1.13.1_20240618.txt @@ -0,0 +1,175 @@ +accessible-pygments==0.0.5 +affine==2.4.0 +alabaster==0.7.16 +asciitree==0.3.3 +astroid==3.2.2 +astropy==6.1.1 +astropy-iers-data==0.2024.6.17.0.31.35 +asttokens==2.4.1 +attrs==23.2.0 +Babel==2.15.0 +bandit==1.7.9 +beautifulsoup4==4.12.3 +black==24.4.2 +CacheControl==0.13.1 +Cartopy==0.23.0 +certifi==2024.6.2 +cftime==1.6.4 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +click-plugins==1.1.1 +cligj==0.7.2 +cloudpickle==3.0.0 +colorama==0.4.6 +configobj==5.0.8 +contourpy==1.2.1 +coverage==7.5.3 +crashtest==0.4.1 +cycler==0.12.1 +dask==2024.6.0 +decorator==5.1.1 +dill==0.3.8 +distlib==0.3.8 +doc8==1.1.1 +docutils==0.20.1 +donfig==0.8.1.post1 +dulwich==0.21.7 +ephem==4.1.5 +exceptiongroup==1.2.1 +executing==2.0.1 +fasteners==0.19 +fastjsonschema==2.19.1 +filelock==3.13.1 +flake8==7.1.0 +flake8-docstrings==1.7.0 +flake8-rst==0.8.0 +flake8-rst-docstrings==0.3.0 +fonttools==4.53.0 +fsspec==2024.6.0 +h5py==3.11.0 +idna==3.7 +imageio==2.34.1 +imagesize==1.4.1 +importlib_metadata==7.1.0 +iniconfig==2.0.0 +installer==0.7.0 +ipython==8.25.0 +isodate==0.6.1 +isort==5.13.2 +jaraco.classes==3.3.1 +jedi==0.19.1 +jeepney==0.8.0 +Jinja2==3.1.4 +jsonschema==4.22.0 +jsonschema-specifications==2023.12.1 +keyring==24.3.0 +kiwisolver==1.4.5 +lazy_loader==0.4 +locket==1.0.0 +m2r2==0.3.3.post2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +matplotlib==3.9.0 +matplotlib-inline==0.1.7 +mccabe==0.7.0 +mdurl==0.1.2 +mistune==0.8.4 +more-itertools==10.2.0 +msgpack==1.0.7 +mypy-extensions==1.0.0 +netCDF4==1.7.1 +networkx==3.3 +numcodecs==0.12.1 +numexpr==2.8.4 +numpy==1.26.4 +packaging==24.1 +pandas==2.2.2 +parso==0.8.4 +partd==1.4.2 +pathspec==0.12.1 +pbr==6.0.0 +pexpect==4.9.0 +pillow==10.3.0 +pipenv==2023.12.1 +pixelmatch==0.3.0 +pkginfo==1.9.6 +platformdirs==3.11.0 +pluggy==1.5.0 +poetry==1.7.1 +poetry-core==1.8.1 +poetry-plugin-export==1.6.0 +pooch==1.8.2 +prettier==0.0.7 +prompt_toolkit==3.0.47 +psutil==5.9.8 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyaml-env==1.2.1 +pycodestyle==2.12.0 +pydata-sphinx-theme==0.15.3 +pydocstyle==6.3.0 +pyerfa==2.0.1.4 +pyflakes==3.2.0 +Pygments==2.18.0 +pyhdf==0.11.4 +pykdtree==1.3.12 +pylint==3.2.3 +pyorbital==1.8.2 +pyparsing==3.1.2 +pyproj==3.6.1 +pyPublicDecompWT==2.8.1.5 +pyresample==1.28.3 +pyshp==2.3.1 +pytest==8.2.2 +pytest-cov==5.0.0 +python-dateutil==2.9.0.post0 +pytz==2024.1 +PyYAML==6.0.1 +rapidfuzz==3.6.1 +rasterio==1.3.10 +referencing==0.35.1 +requests==2.32.3 +requests-toolbelt==1.0.0 +restructuredtext-lint==1.4.0 +rich==13.7.1 +rpds-py==0.18.1 +satpy==0.49.0 +scikit-image==0.23.2 +scipy==1.13.1 +SecretStorage==3.3.3 +shapely==2.0.4 +shellingham==1.5.4 +six==1.16.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +soupsieve==2.5 +Sphinx==7.1.2 +sphinx_design==0.6.0 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-autoprogram==0.1.9 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +stack-data==0.6.3 +stevedore==5.2.0 +tabulate==0.9.0 +tifffile==2024.5.22 +tomli==2.0.1 +tomlkit==0.12.5 +toolz==0.12.1 +traitlets==5.14.3 +trollimage==1.23.2 +trollsift==0.5.1 +trove-classifiers==2024.1.31 +typing_extensions==4.12.2 +tzdata==2024.1 +urllib3==2.2.2 +virtualenv==20.26.2 +wcwidth==0.2.13 +xarray==2024.2.0 +xarray-datatree==0.0.14 +zarr==2.18.2 +zipp==3.19.2 diff --git a/environments/pip_base_requirements_1.13.1_20240704.txt b/environments/pip_base_requirements_1.13.1_20240704.txt new file mode 100644 index 000000000..6ff869b99 --- /dev/null +++ b/environments/pip_base_requirements_1.13.1_20240704.txt @@ -0,0 +1,155 @@ +accessible-pygments==0.0.5 +affine==2.4.0 +alabaster==0.7.16 +alphashape==1.3.1 +asciitree==0.3.3 +astroid==3.2.2 +astropy==6.1.1 +astropy-iers-data==0.2024.7.1.0.34.3 +asttokens==2.4.1 +attrs==23.2.0 +Babel==2.15.0 +bandit==1.7.9 +beautifulsoup4==4.12.3 +black==24.4.2 +Cartopy==0.23.0 +certifi==2024.7.4 +cftime==1.6.4 +charset-normalizer==3.3.2 +click==8.1.7 +click-log==0.4.0 +click-plugins==1.1.1 +cligj==0.7.2 +cloudpickle==3.0.0 +colorama==0.4.6 +configobj==5.0.8 +contourpy==1.2.1 +coverage==7.5.4 +cycler==0.12.1 +dask==2024.6.2 +decorator==5.1.1 +dill==0.3.8 +doc8==1.1.1 +docutils==0.20.1 +donfig==0.8.1.post1 +ephem==4.1.5 +exceptiongroup==1.2.1 +executing==2.0.1 +fasteners==0.19 +flake8==7.1.0 +flake8-docstrings==1.7.0 +flake8-rst==0.8.0 +flake8-rst-docstrings==0.3.0 +fonttools==4.53.0 +fsspec==2024.6.1 +h5py==3.11.0 +idna==3.7 +imageio==2.34.2 +imagesize==1.4.1 +importlib_metadata==8.0.0 +iniconfig==2.0.0 +ipython==8.26.0 +isodate==0.6.1 +isort==5.13.2 +jedi==0.19.1 +Jinja2==3.1.4 +jsonschema==4.22.0 +jsonschema-specifications==2023.12.1 +kiwisolver==1.4.5 +lazy_loader==0.4 +locket==1.0.0 +m2r2==0.3.3.post2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +matplotlib==3.9.1 +matplotlib-inline==0.1.7 +mccabe==0.7.0 +mdurl==0.1.2 +mistune==0.8.4 +mypy-extensions==1.0.0 +netCDF4==1.6.5 +networkx==3.3 +numcodecs==0.12.1 +numexpr==2.8.4 +numpy==1.26.4 +packaging==24.1 +pandas==2.2.2 +parso==0.8.4 +partd==1.4.2 +pathspec==0.12.1 +pbr==6.0.0 +pexpect==4.9.0 +pillow==10.4.0 +pixelmatch==0.3.0 +platformdirs==4.2.2 +pluggy==1.5.0 +pooch==1.8.2 +prettier==0.0.7 +prompt_toolkit==3.0.47 +psutil==6.0.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyaml-env==1.2.1 +pycodestyle==2.12.0 +pydata-sphinx-theme==0.15.4 +pydocstyle==6.3.0 +pyerfa==2.0.1.4 +pyflakes==3.2.0 +Pygments==2.18.0 +pyhdf==0.11.4 +pykdtree==1.3.12 +pylint==3.2.5 +pyorbital==1.8.3 +pyparsing==3.1.2 +pyproj==3.6.1 +pyPublicDecompWT==2.8.1.5 +pyresample==1.28.4 +pyshp==2.3.1 +pytest==8.2.2 +pytest-cov==5.0.0 +python-dateutil==2.9.0.post0 +pytz==2024.1 +PyYAML==6.0.1 +rasterio==1.3.10 +referencing==0.35.1 +requests==2.32.3 +restructuredtext_lint==1.4.0 +rich==13.7.1 +rpds-py==0.18.1 +Rtree==1.2.0 +satpy==0.49.0 +scikit-image==0.24.0 +scipy==1.14.0 +shapely==2.0.4 +six==1.16.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +soupsieve==2.5 +Sphinx==7.1.2 +sphinx_design==0.6.0 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-autoprogram==0.1.9 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +stack-data==0.6.3 +stevedore==5.2.0 +tabulate==0.9.0 +tifffile==2024.7.2 +tomli==2.0.1 +tomlkit==0.12.5 +toolz==0.12.1 +traitlets==5.14.3 +trimesh==4.4.1 +trollimage==1.24.0 +trollsift==0.5.1 +typing_extensions==4.12.2 +tzdata==2024.1 +urllib3==2.2.2 +wcwidth==0.2.13 +xarray==2024.2.0 +xarray-datatree==0.0.14 +zarr==2.18.2 +zipp==3.19.2 diff --git a/environments/pip_base_requirements_1.14.0_20240915.txt b/environments/pip_base_requirements_1.14.0_20240915.txt new file mode 100644 index 000000000..9d1f44a1e --- /dev/null +++ b/environments/pip_base_requirements_1.14.0_20240915.txt @@ -0,0 +1,196 @@ +# Package: geoips_base +# Total run time: 156 seconds +# Number data types run: 3 +# Number data types failed: 0 +# Tue Sep 17 01:45:03 UTC 2024 +accessible-pygments==0.0.5 +affine==2.4.0 +alabaster==0.7.16 +alphashape==1.3.1 +annotated-types==0.7.0 +asciitree==0.3.3 +astroid==3.2.4 +astropy==6.1.3 +astropy-iers-data==0.2024.9.12.13.29.57 +asttokens==2.4.1 +attrs==24.2.0 +babel==2.16.0 +bandit==1.7.9 +beautifulsoup4==4.12.3 +black==24.8.0 +brassy==0.0.2.post2 +CacheControl==0.13.1 +Cartopy==0.23.0 +certifi==2024.8.30 +cffi==1.17.1 +cftime==1.6.4 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +click-log==0.4.0 +click-plugins==1.1.1 +cligj==0.7.2 +cloudpickle==3.0.0 +colorama==0.4.6 +configobj==5.0.8 +contourpy==1.3.0 +coverage==7.6.1 +crashtest==0.4.1 +cycler==0.12.1 +dask==2024.9.0 +decorator==5.1.1 +dill==0.3.8 +distlib==0.3.8 +doc8==1.1.2 +docutils==0.20.1 +donfig==0.8.1.post1 +dulwich==0.21.7 +ephem==4.1.5 +exceptiongroup==1.2.2 +executing==2.1.0 +fasteners==0.19 +fastjsonschema==2.19.1 +filelock==3.13.1 +flake8==7.1.1 +flake8-docstrings==1.7.0 +flake8-rst==0.8.0 +flake8-rst-docstrings==0.3.0 +fonttools==4.53.1 +fsspec==2024.9.0 +h5py==3.11.0 +idna==3.9 +imageio==2.35.1 +imagesize==1.4.1 +importlib_metadata==8.5.0 +iniconfig==2.0.0 +installer==0.7.0 +ipython==8.27.0 +isodate==0.6.1 +isort==5.13.2 +jaraco.classes==3.3.1 +jedi==0.19.1 +jeepney==0.8.0 +Jinja2==3.1.4 +jsonschema==4.23.0 +jsonschema-specifications==2023.12.1 +keyring==24.3.0 +kiwisolver==1.4.7 +lazy_loader==0.4 +locket==1.0.0 +m2r2==0.3.3.post2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +matplotlib==3.9.2 +matplotlib-inline==0.1.7 +mccabe==0.7.0 +mdurl==0.1.2 +mistune==0.8.4 +more-itertools==10.2.0 +morecantile==5.4.2 +msgpack==1.0.7 +mypy-extensions==1.0.0 +netCDF4==1.7.1.post2 +networkx==3.3 +numcodecs==0.13.0 +numexpr==2.8.4 +numpy==1.26.4 +packaging==24.1 +pandas==2.2.2 +parso==0.8.4 +partd==1.4.2 +pathspec==0.12.1 +pbr==6.1.0 +pexpect==4.9.0 +pillow==10.4.0 +pinkrst==0.0.1 +pipenv==2023.12.1 +pixelmatch==0.3.0 +pkginfo==1.9.6 +platformdirs==4.3.3 +pluggy==1.5.0 +poetry==1.7.1 +poetry-core==1.8.1 +poetry-plugin-export==1.6.0 +pooch==1.8.2 +prettier==0.0.7 +prompt_toolkit==3.0.47 +psutil==6.0.0 +ptyprocess==0.7.0 +pure_eval==0.2.3 +pyaml-env==1.2.1 +pycodestyle==2.12.1 +pycparser==2.22 +pydantic==2.9.1 +pydantic_core==2.23.3 +pydata-sphinx-theme==0.15.4 +pydocstyle==6.3.0 +pyerfa==2.0.1.4 +pyflakes==3.2.0 +pygit2==1.15.1 +Pygments==2.18.0 +pyhdf==0.11.4 +pykdtree==1.3.13 +pylint==3.2.7 +pyorbital==1.8.3 +pyparsing==3.1.4 +pyproj==3.6.1 +pyPublicDecompWT==2.8.1.6 +pyresample==1.30.0 +pyshp==2.3.1 +pytest==8.3.3 +pytest-cov==5.0.0 +python-dateutil==2.9.0.post0 +pytz==2024.2 +PyYAML==6.0.2 +rapidfuzz==3.6.1 +rasterio==1.3.11 +referencing==0.35.1 +requests==2.32.3 +requests-toolbelt==1.0.0 +restructuredtext-lint==1.4.0 +rich==13.8.1 +rich-argparse==1.5.2 +rio-cogeo==5.3.4 +rpds-py==0.20.0 +Rtree==1.3.0 +satpy==0.51.0 +scikit-image==0.24.0 +scipy==1.14.1 +SecretStorage==3.3.3 +shapely==2.0.6 +shellingham==1.5.4 +six==1.16.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +soupsieve==2.6 +Sphinx==7.1.2 +sphinx-argparse==0.5.2 +sphinx_design==0.6.1 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-autoprogram==0.1.9 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 +stack-data==0.6.3 +stevedore==5.3.0 +tabulate==0.9.0 +tifffile==2024.8.30 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +traitlets==5.14.3 +trimesh==4.4.9 +trollimage==1.25.0 +trollsift==0.5.1 +trove-classifiers==2024.1.31 +typing_extensions==4.12.2 +tzdata==2024.1 +urllib3==2.2.3 +virtualenv==20.26.3 +wcwidth==0.2.13 +xarray==2024.2.0 +xarray-datatree==0.0.14 +zarr==2.18.3 +zipp==3.20.2 diff --git a/environments/pip_full_requirements_1.14.0_20240916.txt b/environments/pip_full_requirements_1.14.0_20240916.txt new file mode 100644 index 000000000..097204d35 --- /dev/null +++ b/environments/pip_full_requirements_1.14.0_20240916.txt @@ -0,0 +1,197 @@ +# Package: geoips_full +# Total run time: 2558 seconds +# Number data types run: 56 +# Number data types failed: 0 +# Mon Sep 16 22:06:05 UTC 2024 +accessible-pygments==0.0.5 +affine==2.4.0 +# Editable install with no version control (akima86==0.1.1) +alabaster==0.7.16 +alphashape==1.3.1 +annotated-types==0.7.0 +asciitree==0.3.3 +astroid==3.2.4 +astropy==6.1.3 +astropy-iers-data==0.2024.9.12.13.29.57 +asttokens==2.4.1 +attrs==24.2.0 +babel==2.16.0 +bandit==1.7.9 +beautifulsoup4==4.12.3 +black==24.8.0 +brassy==0.0.2.post2 +CacheControl==0.13.1 +Cartopy==0.23.0 +certifi==2024.8.30 +cffi==1.17.1 +cftime==1.6.4 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +click-log==0.4.0 +click-plugins==1.1.1 +cligj==0.7.2 +cloudpickle==3.0.0 +colorama==0.4.6 +configobj==5.0.8 +contourpy==1.3.0 +coverage==7.6.1 +crashtest==0.4.1 +cycler==0.12.1 +dask==2024.9.0 +decorator==5.1.1 +dill==0.3.8 +distlib==0.3.8 +doc8==1.1.2 +docutils==0.20.1 +donfig==0.8.1.post1 +dulwich==0.21.7 +ephem==4.1.5 +exceptiongroup==1.2.2 +executing==2.1.0 +fasteners==0.19 +fastjsonschema==2.19.1 +filelock==3.13.1 +flake8==7.1.1 +flake8-docstrings==1.7.0 +flake8-rst==0.8.0 +flake8-rst-docstrings==0.3.0 +fonttools==4.53.1 +fsspec==2024.9.0 +h5py==3.11.0 +idna==3.9 +imageio==2.35.1 +imagesize==1.4.1 +importlib_metadata==8.5.0 +iniconfig==2.0.0 +installer==0.7.0 +ipython==8.27.0 +isodate==0.6.1 +isort==5.13.2 +jaraco.classes==3.3.1 +jedi==0.19.1 +jeepney==0.8.0 +Jinja2==3.1.4 +jsonschema==4.23.0 +jsonschema-specifications==2023.12.1 +keyring==24.3.0 +kiwisolver==1.4.7 +lazy_loader==0.4 +locket==1.0.0 +m2r2==0.3.3.post2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +matplotlib==3.9.2 +matplotlib-inline==0.1.7 +mccabe==0.7.0 +mdurl==0.1.2 +mistune==0.8.4 +more-itertools==10.2.0 +morecantile==5.4.2 +msgpack==1.0.7 +mypy-extensions==1.0.0 +netCDF4==1.7.1.post2 +networkx==3.3 +numcodecs==0.13.0 +numexpr==2.8.4 +numpy==1.26.4 +packaging==24.1 +pandas==2.2.2 +parso==0.8.4 +partd==1.4.2 +pathspec==0.12.1 +pbr==6.1.0 +pexpect==4.9.0 +pillow==10.4.0 +pinkrst==0.0.1 +pipenv==2023.12.1 +pixelmatch==0.3.0 +pkginfo==1.9.6 +platformdirs==4.3.3 +pluggy==1.5.0 +poetry==1.7.1 +poetry-core==1.8.1 +poetry-plugin-export==1.6.0 +pooch==1.8.2 +prettier==0.0.7 +prompt_toolkit==3.0.47 +psutil==6.0.0 +ptyprocess==0.7.0 +pure_eval==0.2.3 +pyaml-env==1.2.1 +pycodestyle==2.12.1 +pycparser==2.22 +pydantic==2.9.1 +pydantic_core==2.23.3 +pydata-sphinx-theme==0.15.4 +pydocstyle==6.3.0 +pyerfa==2.0.1.4 +pyflakes==3.2.0 +pygit2==1.15.1 +Pygments==2.18.0 +pyhdf==0.11.4 +pykdtree==1.3.13 +pylint==3.2.7 +pyorbital==1.8.3 +pyparsing==3.1.4 +pyproj==3.6.1 +pyPublicDecompWT==2.8.1.6 +pyresample==1.30.0 +pyshp==2.3.1 +pytest==8.3.3 +pytest-cov==5.0.0 +python-dateutil==2.9.0.post0 +pytz==2024.2 +PyYAML==6.0.2 +rapidfuzz==3.6.1 +rasterio==1.3.11 +referencing==0.35.1 +requests==2.32.3 +requests-toolbelt==1.0.0 +restructuredtext-lint==1.4.0 +rich==13.8.1 +rich-argparse==1.5.2 +rio-cogeo==5.3.4 +rpds-py==0.20.0 +Rtree==1.3.0 +satpy==0.51.0 +scikit-image==0.24.0 +scipy==1.14.1 +SecretStorage==3.3.3 +shapely==2.0.6 +shellingham==1.5.4 +six==1.16.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +soupsieve==2.6 +Sphinx==7.1.2 +sphinx-argparse==0.5.2 +sphinx_design==0.6.1 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-autoprogram==0.1.9 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 +stack-data==0.6.3 +stevedore==5.3.0 +tabulate==0.9.0 +tifffile==2024.8.30 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +traitlets==5.14.3 +trimesh==4.4.9 +trollimage==1.25.0 +trollsift==0.5.1 +trove-classifiers==2024.1.31 +typing_extensions==4.12.2 +tzdata==2024.1 +urllib3==2.2.3 +virtualenv==20.26.3 +wcwidth==0.2.13 +xarray==2024.2.0 +xarray-datatree==0.0.14 +zarr==2.18.3 +zipp==3.20.2 diff --git a/geoips/commandline/geoips_tree.py b/geoips/commandline/geoips_tree.py index 946c980bb..5881e094b 100644 --- a/geoips/commandline/geoips_tree.py +++ b/geoips/commandline/geoips_tree.py @@ -1,3 +1,6 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + """GeoIPS CLI "tree" command. Single 'geoips tree' command which will display GeoIPS CLI commands up to a --max-depth diff --git a/geoips/commandline/log_setup.py b/geoips/commandline/log_setup.py index 8f85cccb1..d34f6bfd5 100644 --- a/geoips/commandline/log_setup.py +++ b/geoips/commandline/log_setup.py @@ -8,10 +8,6 @@ from textwrap import wrap -logging_setup_done = False -log = None - - def log_with_emphasis(print_func, *messages): """Print messages boxed in asterisks using the specified print function. @@ -190,12 +186,6 @@ def setup_logging(logging_level=None, verbose=True): instance via `LOG = logging.getLogger(__name__)` and it will behave the same as the root logger. - If this function was called previously in the program's traceback in any other - location, then setup_logging will default to the log setup in the very first call. - This is done so log statements aren't duplicated the number of times this function - has been called. Therefore, it is smart to ensure that the first call to this - function is the lowest log level you'ld like to capture. - Parameters ---------- logging_level : str, default=None @@ -206,34 +196,21 @@ def setup_logging(logging_level=None, verbose=True): Determines which log formatter will be used. If `True`, a longer format will be used, providing more information, but also cluttering the screen. If `False`, a shorter format will be used. - - Globals - ------- - logging_setup_done: bool - - Whether or not this function has already been called - log: logging.RootLogger - - A root logging object that has the GeoIPS log format applied. This will only - be set once in a program's execution. """ - global logging_setup_done, log - - if not logging_setup_done: - log = logging.getLogger() - # If logging_level was not specified, default to INTERACTIVE here. - if not logging_level: - logging_level = "INTERACTIVE" - log.setLevel(getattr(logging, logging_level)) - fmt = logging.Formatter( - "%(asctime)s %(module)12s.py:%(lineno)-4d %(levelname)7s: %(message)s", - "%d_%H%M%S", - ) - if not verbose: - fmt = logging.Formatter("%(asctime)s: %(message)s", "%d_%H%M%S") + log = logging.getLogger() + # If logging_level was not specified, default to INTERACTIVE here. + if not logging_level: + logging_level = "INTERACTIVE" + log.setLevel(getattr(logging, logging_level)) + fmt = logging.Formatter( + "%(asctime)s %(module)12s.py:%(lineno)-4d %(levelname)7s: %(message)s", + "%d_%H%M%S", + ) + if not verbose: + fmt = logging.Formatter("%(asctime)s: %(message)s", "%d_%H%M%S") + if not log.handlers: stream_hndlr = logging.StreamHandler(sys.stdout) stream_hndlr.setFormatter(fmt) stream_hndlr.setLevel(logging.INFO) log.addHandler(stream_hndlr) - - logging_setup_done = True - return log diff --git a/geoips/dev/product.py b/geoips/dev/product.py index 396809197..402a6ec6d 100644 --- a/geoips/dev/product.py +++ b/geoips/dev/product.py @@ -140,6 +140,32 @@ def get_data_range(prod_plugin, output_dict=None): return alg_args["output_data_range"].copy() +def get_cmap_data_range(prod_plugin, output_dict=None): + """Interface will be deprecated v2.0. + + Retrieve required data range for requested product + + Parameters + ---------- + product_name : str + Name of requested product (ie, 'IR-BD', '89H', 'color89Nearest', etc) + + Returns + ------- + data_range : list + List of float specifying min and max value for the output product + ``.colormapper..arguments['data_range']`` + """ + cmap_args = prod_plugin["spec"]["colormapper"]["plugin"]["arguments"] + if "data_range" not in cmap_args: + raise TypeError( + f"Product {prod_plugin.name} does not define 'output_data_range' for its " + f"algorithm." + ) + # Add .copy() so we aren't returning the ACTUAL list attached to the args + return cmap_args["data_range"].copy() + + def get_product_display_name(prod_plugin, output_dict=None): """Interface will be deprecated v2.0. diff --git a/geoips/filenames/base_paths.py b/geoips/filenames/base_paths.py index 306ef2c65..1ff88e5c3 100755 --- a/geoips/filenames/base_paths.py +++ b/geoips/filenames/base_paths.py @@ -92,6 +92,12 @@ PATHS["GEOIPS_OUTDIRS"], "preprocessed", "algorithms" ) +PATHS["OUTPUT_CHECKER_THRESHOLD_IMAGE"] = 0.05 +if getenv("OUTPUT_CHECKER_THRESHOLD_IMAGE"): + PATHS["OUTPUT_CHECKER_THRESHOLD_IMAGE"] = float( + getenv("OUTPUT_CHECKER_THRESHOLD_IMAGE").rstrip("/") + ) + # Location for writing out pregenerated "clean" imagery files if getenv("CLEAN_IMAGERY_PATH"): PATHS["CLEAN_IMAGERY_PATH"] = getenv("CLEAN_IMAGERY_PATH").rstrip("/") diff --git a/geoips/interfaces/module_based/output_checkers.py b/geoips/interfaces/module_based/output_checkers.py index 18f6142b8..a884225be 100644 --- a/geoips/interfaces/module_based/output_checkers.py +++ b/geoips/interfaces/module_based/output_checkers.py @@ -1012,7 +1012,7 @@ def identify_checker(self, filename): return checker_name def get_plugin(self, name): - """Return the output checker plugin corresponding to checker_name.""" + """Get the output checker plugin corresponding to checker_name.""" plug = super().get_plugin(name) if self.valid_plugin(plug): return plug diff --git a/geoips/plugins/modules/colormappers/winds/wind_radii_transitions.py b/geoips/plugins/modules/colormappers/winds/wind_radii_transitions.py index 20aa2ad82..e455de3c1 100644 --- a/geoips/plugins/modules/colormappers/winds/wind_radii_transitions.py +++ b/geoips/plugins/modules/colormappers/winds/wind_radii_transitions.py @@ -37,36 +37,36 @@ def call(data_range=[0, 200]): min_wind_speed = data_range[0] max_wind_speed = data_range[1] transition_vals = [ - (min_wind_speed, 34), + (min_wind_speed, 12), + (12, 25), + (25, 34), (34, 50), (50, 64), (64, 80), - # (64, 72), - # (72, 80), (80, 100), (100, 120), - (120, 150), - (150, max_wind_speed), + (120, max_wind_speed), ] transition_colors = [ - ("lightblue", "blue"), + ("white", "#739FE1"), + ("#3f82ff", "blue"), + ("#94d9a7", "#317E0B"), ("yellow", "orange"), - ("red", "red"), - # ('thistle', 'thistle'), - # ('firebrick', 'firebrick'), - # ('fuchsia', 'fuchsia'), - # ('mediumvioletred', 'mediumvioletred'), - ("rebeccapurple", "rebeccapurple"), - # ('purple', 'rebeccapurple'), - # ('rebeccapurple', 'rebeccapurple'), - # ('mediumvioletred', 'mediumvioletred'), - ("palevioletred", "palevioletred"), - ("silver", "silver"), - ("gray", "gray"), - ("dimgray", "dimgray"), + ("#ff7878", "#C90A0A"), + ("#C285F6", "rebeccapurple"), + ("#fcb4cc", "palevioletred"), + # Grays + # ("#BCB8B8", "#999898"), + # ("#999898", "#808080"), + # ("#808080", "#737373"), + # ("#737373", "#5a5a5a"), + # ("#5a5a5a", "#3d3d3d"), + ("#BCB8B8", "dimgray"), + ("darkslategray", "black"), ] - ticks = [xx[0] for xx in transition_vals] + # ticks = [xx[0] for xx in transition_vals] + ticks = [0, 12, 25, 34, 50, 64, 80, 100, 120, 150, 200] min_wind_speed = transition_vals[0][0] max_wind_speed = transition_vals[-1][1] @@ -104,6 +104,7 @@ def call(data_range=[0, 200]): "boundaries": mpl_boundaries, "cbar_spacing": cbar_spacing, "colorbar": True, + "cbar_full_width": True, } # return cbar, min_wind_speed, max_wind_speed diff --git a/geoips/plugins/modules/colormappers/winds/wind_radii_transitions_legacy.py b/geoips/plugins/modules/colormappers/winds/wind_radii_transitions_legacy.py new file mode 100644 index 000000000..ea92f1aac --- /dev/null +++ b/geoips/plugins/modules/colormappers/winds/wind_radii_transitions_legacy.py @@ -0,0 +1,110 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + +"""Module containing wind speed colormap with transitions at 34, 50, 64, and 80.""" + +import logging + +LOG = logging.getLogger(__name__) + +interface = "colormappers" +family = "matplotlib" +name = "wind_radii_transitions_legacy" + + +def call(data_range=[0, 200]): + """Generate appropriate matplotlib colors for plotting standard wind speeds. + + wind_radii_transitions contains hard coded transition values for different + colors, in order to have consistent imagery across all sensors / products. + + Parameters + ---------- + data_range : list of float, default=[0, 200] + * Min and max value for colormap. + * Ensure the data range matches the range of the algorithm specified for + use with this colormap + * This colormap MUST include 0 and 200 + + Returns + ------- + mpl_colors_info : dict + Dictionary of matplotlib plotting parameters, to ensure consistent + image output + """ + from geoips.image_utils.colormap_utils import create_linear_segmented_colormap + + min_wind_speed = data_range[0] + max_wind_speed = data_range[1] + transition_vals = [ + (min_wind_speed, 34), + (34, 50), + (50, 64), + (64, 80), + # (64, 72), + # (72, 80), + (80, 100), + (100, 120), + (120, 150), + (150, max_wind_speed), + ] + transition_colors = [ + ("lightblue", "blue"), + ("yellow", "orange"), + ("red", "red"), + # ('thistle', 'thistle'), + # ('firebrick', 'firebrick'), + # ('fuchsia', 'fuchsia'), + # ('mediumvioletred', 'mediumvioletred'), + ("rebeccapurple", "rebeccapurple"), + # ('purple', 'rebeccapurple'), + # ('rebeccapurple', 'rebeccapurple'), + # ('mediumvioletred', 'mediumvioletred'), + ("palevioletred", "palevioletred"), + ("silver", "silver"), + ("gray", "gray"), + ("dimgray", "dimgray"), + ] + + ticks = [xx[0] for xx in transition_vals] + + min_wind_speed = transition_vals[0][0] + max_wind_speed = transition_vals[-1][1] + + LOG.info("Setting cmap") + mpl_cmap = create_linear_segmented_colormap( + "windspeed_cmap", + min_wind_speed, + max_wind_speed, + transition_vals, + transition_colors, + ) + + LOG.info("Setting norm") + from matplotlib.colors import Normalize + + mpl_norm = Normalize(vmin=min_wind_speed, vmax=max_wind_speed) + + cbar_label = "Surface Wind (knots)" + + # Must be uniform or proportional, None not valid for Python 3 + cbar_spacing = "proportional" + mpl_tick_labels = None + mpl_boundaries = None + + # from geoips.image_utils.mpl_utils import create_colorbar + # only create colorbar for final imagery + # cbar = create_colorbar(fig, mpl_cmap, mpl_norm, ticks, cbar_label=cbar_label) + mpl_colors_info = { + "cmap": mpl_cmap, + "norm": mpl_norm, + "cbar_ticks": ticks, + "cbar_tick_labels": mpl_tick_labels, + "cbar_label": cbar_label, + "boundaries": mpl_boundaries, + "cbar_spacing": cbar_spacing, + "colorbar": True, + } + + # return cbar, min_wind_speed, max_wind_speed + return mpl_colors_info diff --git a/geoips/plugins/modules/interpolators/utils/interp_scipy.py b/geoips/plugins/modules/interpolators/utils/interp_scipy.py index 7228332d5..7f4aa2c12 100644 --- a/geoips/plugins/modules/interpolators/utils/interp_scipy.py +++ b/geoips/plugins/modules/interpolators/utils/interp_scipy.py @@ -10,6 +10,10 @@ import scipy import numpy +# imports for alphashape-based masking +import alphashape +from skimage.measure import points_in_poly + LOG = logging.getLogger(__name__) # interface = None indicates to the GeoIPS interfaces that this is not a valid @@ -145,12 +149,36 @@ def interp_griddata( interp_data = scipy.interpolate.griddata( (data_lats, data_lons), data_array, (gridlats, gridlons), method=method ) + list_of_lons = numpy.reshape(gridlons, -1) + list_of_lats = numpy.reshape(gridlats, -1) + # Free up memory ?? gridlons = 1 gridlats = 1 + # Zip data_lons and data_lats and compute alphashape + datapts_2d = list(zip(data_lons.data, data_lats.data)) + returned_polygons = alphashape.alphashape(datapts_2d, 1.0) + # If multiple polygons are returned, choose the one with the largest area + if returned_polygons.geom_type == "MultiPolygon": + alpha_shape = max(returned_polygons.geoms, key=lambda a: a.area) + elif returned_polygons.geom_type == "Polygon": + alpha_shape = returned_polygons + interp_data = numpy.ma.masked_invalid(interp_data) interp_data = numpy.ma.masked_less(interp_data, data_array.min()) interp_data = numpy.ma.masked_greater(interp_data, data_array.max()) + input_shape_interp_data = interp_data.shape + + zipped_lons_lats = numpy.asarray(list(zip(list_of_lons, list_of_lats))) + mask_exterior_coords_xy = numpy.asarray(list(zip(*alpha_shape.exterior.coords.xy))) + inside_hull_mask = numpy.reshape( + points_in_poly(zipped_lons_lats, mask_exterior_coords_xy), + input_shape_interp_data, + ) + + interp_data = numpy.ma.masked_where( + numpy.logical_not(inside_hull_mask), interp_data + ) return interp_data diff --git a/geoips/plugins/modules/output_checkers/image.py b/geoips/plugins/modules/output_checkers/image.py index fe243b933..2d4b7e432 100644 --- a/geoips/plugins/modules/output_checkers/image.py +++ b/geoips/plugins/modules/output_checkers/image.py @@ -6,6 +6,8 @@ import logging from os.path import splitext +from geoips.filenames.base_paths import PATHS as gpaths + LOG = logging.getLogger(__name__) interface = "output_checkers" @@ -84,7 +86,12 @@ def correct_file_format(fname): return False -def outputs_match(plugin, output_product, compare_product, threshold): +def outputs_match( + plugin, + output_product, + compare_product, + threshold=gpaths["OUTPUT_CHECKER_THRESHOLD_IMAGE"], +): """Use PIL and numpy to compare two images. Parameters @@ -95,7 +102,7 @@ def outputs_match(plugin, output_product, compare_product, threshold): Current output product compare_product : str Path to comparison product - threshold: float + threshold: float, default=0.05 Threshold for the image comparison. Argument to pixelmatch. Between 0 and 1, with 0 the most strict comparison, and 1 the most lenient. @@ -234,7 +241,12 @@ def outputs_match(plugin, output_product, compare_product, threshold): return True -def call(plugin, compare_path, output_products, threshold=0.05): +def call( + plugin, + compare_path, + output_products, + threshold=gpaths["OUTPUT_CHECKER_THRESHOLD_IMAGE"], +): """Compare the "correct" imagery found the list of current output_products. Compares files produced in the current processing run with the list of diff --git a/geoips/plugins/modules/output_formatters/histogram_csv.py b/geoips/plugins/modules/output_formatters/histogram_csv.py new file mode 100644 index 000000000..bd1d3a4cd --- /dev/null +++ b/geoips/plugins/modules/output_formatters/histogram_csv.py @@ -0,0 +1,62 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + +"""Produce histogram from the given dataset with specified bin size.""" +import logging +import numpy + +LOG = logging.getLogger(__name__) + +interface = "output_formatters" +family = "xarray_data" +name = "histogram_csv" + + +def call( + xarray_obj, product_names, output_fnames, min_val=None, max_val=None, num_bins=501 +): + """Produce histogram. + + Parameters + ---------- + xarray_obj: xarray Dataset + + Returns + ------- + xarray.Dataset + Resulting histogram + """ + import xarray + + prod_xarray = xarray.Dataset() + + from geoips.geoips_utils import copy_standard_metadata + from geoips.filenames.base_paths import make_dirs + from os.path import dirname + + copy_standard_metadata(xarray_obj, prod_xarray) + for product_name in product_names: + prod_xarray[product_name] = xarray_obj[product_name] + + if product_name == "latitude" or product_name == "longitude": + continue + xda = xarray_obj[product_name] + hist = numpy.histogram(xda, numpy.linspace(min_val, max_val, num_bins + 1)) + prod_xarray[f"{product_name}_histogram"] = xarray.DataArray( + hist[0], dims=("dim_2") + ) + prod_xarray["bins"] = xarray.DataArray(hist[1], dims=("dim_3")) + for ncdf_fname in output_fnames: + make_dirs(dirname(ncdf_fname)) + with open(ncdf_fname, "w") as fobj: + prod = prod_xarray[f"{product_name}_histogram"].to_numpy() + fobj.write(",".join([str(val) for val in list(prod)])) + LOG.interactive(f"Writing {ncdf_fname}") + + # from geoips.plugins.modules.output_formatters.netcdf_xarray import ( + # write_xarray_netcdf, + # ) + + # for ncdf_fname in output_fnames: + # write_xarray_netcdf(prod_xarray, ncdf_fname) + return output_fnames diff --git a/geoips/plugins/modules/output_formatters/netcdf_geoips.py b/geoips/plugins/modules/output_formatters/netcdf_geoips.py index 2e766eacc..4154cf8fd 100644 --- a/geoips/plugins/modules/output_formatters/netcdf_geoips.py +++ b/geoips/plugins/modules/output_formatters/netcdf_geoips.py @@ -11,7 +11,7 @@ name = "netcdf_geoips" -def call(xarray_obj, product_names, output_fnames): +def call(xarray_obj, product_names, output_fnames, clobber=False): """Write GeoIPS style NetCDF to disk.""" import xarray @@ -28,5 +28,5 @@ def call(xarray_obj, product_names, output_fnames): ) for ncdf_fname in output_fnames: - write_xarray_netcdf(prod_xarray, ncdf_fname) + write_xarray_netcdf(prod_xarray, ncdf_fname, clobber=clobber) return output_fnames diff --git a/geoips/plugins/modules/output_formatters/netcdf_xarray.py b/geoips/plugins/modules/output_formatters/netcdf_xarray.py index 7b1141993..a4007ae8d 100644 --- a/geoips/plugins/modules/output_formatters/netcdf_xarray.py +++ b/geoips/plugins/modules/output_formatters/netcdf_xarray.py @@ -12,7 +12,7 @@ name = "netcdf_xarray" -def call(xarray_obj, product_names, output_fnames): +def call(xarray_obj, product_names, output_fnames, clobber=False): """Write xarray-based NetCDF outputs to disk.""" for ncdf_fname in output_fnames: write_xarray_netcdf(xarray_obj, ncdf_fname) diff --git a/geoips/plugins/modules/procflows/config_based.py b/geoips/plugins/modules/procflows/config_based.py index 255c9b732..a8297c1bf 100644 --- a/geoips/plugins/modules/procflows/config_based.py +++ b/geoips/plugins/modules/procflows/config_based.py @@ -756,6 +756,23 @@ def get_area_defs_from_available_sectors( if sector_dict.get("trackfiles"): sector_dict["tcdb"] = False + # Check if sector_list specified under YAML output config file is a list or a + # dictionary. If sector_list is a list, static sectors are enabled for all + # platforms that use the output config YAML. If sector_list is a dictionary, + # each key is a platform name that holds a list of static sectors to be + # processed for said platform. If sector_list is a dictionary, and the platform + # name is not a key, warning is raised and sector_list is set as an empty list. + if sector_dict.get("sector_list") and isinstance( + sector_dict.get("sector_list"), dict + ): + try: + sector_dict["sector_list"] = sector_dict["sector_list"][ + xobjs["METADATA"].platform_name + ] + except KeyError as resp: + LOG.warning("%s MISSING PLATFORM NAME", resp) + sector_dict["sector_list"] = [] + # This is the standard "get_area_defs_from_command_line_args", YAML config # specified sector information matches the command line specified sector # information diff --git a/geoips/plugins/modules/readers/cygnss_netcdf.py b/geoips/plugins/modules/readers/cygnss_netcdf.py new file mode 100644 index 000000000..53611081a --- /dev/null +++ b/geoips/plugins/modules/readers/cygnss_netcdf.py @@ -0,0 +1,114 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + +"""Read derived surface winds from CYGNSS netcdf data.""" + +import glob +import logging +import xarray + +from geoips.xarray_utils.time import get_min_from_xarray_time, get_max_from_xarray_time + +LOG = logging.getLogger(__name__) + +MS_TO_KTS = 1.94384 +DEG_TO_KM = 111.321 + +interface = "readers" +family = "standard" +name = "cygnss_netcdf" + + +def call(fnames, metadata_only=False, chans=None, area_def=None, self_register=False): + """Read CYGNSS derived winds from netcdf data. + + Parameters + ---------- + fnames : list + * List of strings, full paths to files + metadata_only : bool, default=False + * Return before actually reading data if True + chans : list of str, default=None + * List of desired channels (skip unneeded variables as needed). + * Include all channels if None. + area_def : pyresample.AreaDefinition, default=None + * NOT YET IMPLEMENTED + * Specify region to read + * Read all data if None. + self_register : str or bool, default=False + * NOT YET IMPLEMENTED + * register all data to the specified dataset id (as specified in the + return dictionary keys). + * Read multiple resolutions of data if False. + + Returns + ------- + dict of xarray.Datasets + * dictionary of xarray.Dataset objects with required Variables and + Attributes. + * Dictionary keys can be any descriptive dataset ids. + + See Also + -------- + :ref:`xarray_standards` + Additional information regarding required attributes and variables + for GeoIPS-formatted xarray Datasets. + """ + out_dict = {} + + # Single file is full day of data, so only need one. + fname = fnames[0] + + wind_xarray = xarray.open_dataset(str(fname)) + wind_xarray = wind_xarray.rename( + {"lat": "latitude", "lon": "longitude", "sample_time": "time"} + ) + wind_xarray.attrs["source_name"] = "cygnss" + wind_xarray.attrs["platform_name"] = "cygnss" + wind_xarray.attrs["data_provider"] = "noaa-nesdis-star-socd-oswt" + wind_xarray.attrs["source_file_names"] = fnames + wind_xarray.attrs["interpolation_radius_of_influence"] = 20000 + wind_xarray.attrs["sample_distance_km"] = DEG_TO_KM / 4 + + wind_xarray.attrs["start_datetime"] = get_min_from_xarray_time(wind_xarray, "time") + wind_xarray.attrs["end_datetime"] = get_max_from_xarray_time(wind_xarray, "time") + + out_dict["METADATA"] = wind_xarray[[]] + if metadata_only: + return out_dict + + if chans: + if "wind_speed_kts" in chans: + wind_xarray["wind_speed_kts"] = wind_xarray["wind_speed"] * MS_TO_KTS + wind_xarray["wind_speed_kts"].attrs["units"] = "kts" + + ds_var_list = list(wind_xarray.variables.keys()) + + if chans: + for chan in chans: + if chan not in ds_var_list: + raise ValueError(f"Variable {chan} not found in dataset.") + + wind_xarray = wind_xarray[chans] + + out_dict["CYGNSS"] = wind_xarray + + return out_dict + + +# Unit test functions +def get_test_files(test_data_dir): + """Generate testing xarray from test data.""" + filepath = test_data_dir + "/test_data_cygnss/data/*.nc" + filelist = glob.glob(filepath) + tmp_xr = call(filelist) + if len(filelist) == 0: + raise NameError("No files found") + return tmp_xr + + +def get_test_parameters(): + """Generate test data key for unit testing.""" + return [ + {"data_key": "CYGNSS", "data_var": "wind_speed", "mean": 7.0251474}, + ] diff --git a/geoips/plugins/modules/readers/scat_knmi_winds_netcdf.py b/geoips/plugins/modules/readers/scat_knmi_winds_netcdf.py index 3a5143555..bb168bba7 100644 --- a/geoips/plugins/modules/readers/scat_knmi_winds_netcdf.py +++ b/geoips/plugins/modules/readers/scat_knmi_winds_netcdf.py @@ -43,6 +43,9 @@ def read_knmi_data(wind_xarray): elif wind_xarray.source == "ScatSat-1 OSCAT": geoips_metadata["source_name"] = "oscat" geoips_metadata["platform_name"] = "scatsat-1" + elif wind_xarray.source == "Oceansat-3 OSCAT": + geoips_metadata["source_name"] = "oscat" + geoips_metadata["platform_name"] = "oceansat-3" elif wind_xarray.source == "HY-2D HSCAT": geoips_metadata["source_name"] = "hscat" geoips_metadata["platform_name"] = "hy-2d" diff --git a/geoips/plugins/modules/readers/viirs_sdr_hdf5.py b/geoips/plugins/modules/readers/viirs_sdr_hdf5.py index 07c36e91b..d009d99f5 100755 --- a/geoips/plugins/modules/readers/viirs_sdr_hdf5.py +++ b/geoips/plugins/modules/readers/viirs_sdr_hdf5.py @@ -1,3 +1,6 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + """VIIRS SDR Satpy reader. This VIIRS reader is designed for reading the NPP/JPSS SDR HDF5 files. diff --git a/geoips/plugins/yaml/product_defaults/pmw_89/89pct.yaml b/geoips/plugins/yaml/product_defaults/pmw_89/89pct.yaml index 23f193bb5..d40e3161e 100644 --- a/geoips/plugins/yaml/product_defaults/pmw_89/89pct.yaml +++ b/geoips/plugins/yaml/product_defaults/pmw_89/89pct.yaml @@ -18,7 +18,7 @@ spec: plugin: name: pmw_89pct arguments: - data_range: [105, 280] + data_range: [105.0, 280.0] interpolator: plugin: name: interp_gauss diff --git a/geoips/plugins/yaml/products/ascat.yaml b/geoips/plugins/yaml/products/ascat.yaml index 494f9eaa7..f7dc41254 100644 --- a/geoips/plugins/yaml/products/ascat.yaml +++ b/geoips/plugins/yaml/products/ascat.yaml @@ -46,7 +46,8 @@ spec: - name: wind-ambiguities source_names: [ascat] docstring: | - The wind-ambiguities product_defaults for ascat product. + Scatterometer data that shows and abstract of possible wind + direction. Usually have four cases so the first two shown are most likely. product_defaults: wind-ambiguities spec: variables: diff --git a/geoips/plugins/yaml/products/cygnss.yaml b/geoips/plugins/yaml/products/cygnss.yaml new file mode 100644 index 000000000..2656ae61d --- /dev/null +++ b/geoips/plugins/yaml/products/cygnss.yaml @@ -0,0 +1,15 @@ +interface: products +family: list +name: cygnss +docstring: | + The cygnss products configuration, which produces Windspeed products, Windbarbs products, + Wind-ambiguity products, and Unsectored/Sectored products. +spec: + products: + - name: windspeed + source_names: [cygnss] + docstring: | + The windspeed product_defaults for cygnss product. + product_defaults: windspeed + spec: + variables: ["wind_speed_kts"] \ No newline at end of file diff --git a/geoips/plugins/yaml/products/oscat.yaml b/geoips/plugins/yaml/products/oscat.yaml index b8124ff66..c4e4de3b6 100644 --- a/geoips/plugins/yaml/products/oscat.yaml +++ b/geoips/plugins/yaml/products/oscat.yaml @@ -31,3 +31,17 @@ spec: "wind_dir_deg_ambiguity_met", "rain_flag_ambiguity", ] + - name: unsectored + source_names: [oscat] + docstring: | + The unsectored product_defaults for oscat product. + product_defaults: unsectored + spec: + variables: ["wind_speed_kts", "wind_dir_deg_met"] + - name: sectored + source_names: [oscat] + docstring: | + The sectored product_defaults for oscat product. + product_defaults: sectored + spec: + variables: ["wind_speed_kts", "wind_dir_deg_met"] diff --git a/geoips/plugins/yaml/products/smos-spd.yaml b/geoips/plugins/yaml/products/smos-spd.yaml index b763fa5da..2c5443510 100644 --- a/geoips/plugins/yaml/products/smos-spd.yaml +++ b/geoips/plugins/yaml/products/smos-spd.yaml @@ -2,6 +2,8 @@ interface: products family: list name: smos-spd docstring: | + Soil Moisture sensor that “smart people can use for windspeed” (Sampson, Meteorologist). + Wavelength (3 cm) is long enough to penetrate heavy rains but shows low resolution (40 km). The smos-spd product_inputs configuration, which produces windspeed, unsectored, and sectored products. spec: products: diff --git a/geoips/sector_utils/tc_tracks_database.py b/geoips/sector_utils/tc_tracks_database.py index 413101a32..c5bf99db6 100644 --- a/geoips/sector_utils/tc_tracks_database.py +++ b/geoips/sector_utils/tc_tracks_database.py @@ -116,13 +116,17 @@ def update_fields(tc_trackfilename, cc, conn, process=False): # Check if timestamp on file is newer than timestamp in database - # if not, just return and don't do anything. if data: - database_timestamp = datetime.strptime( - cc.execute( - "SELECT last_updated from tc_trackfiles WHERE filename = ?", - (tc_trackfilename,), - ).fetchone()[0], - "%Y-%m-%d %H:%M:%S.%f", - ) + try: + database_timestamp = datetime.strptime( + cc.execute( + "SELECT last_updated from tc_trackfiles WHERE filename = ?", + (tc_trackfilename,), + ).fetchone()[0], + "%Y-%m-%d %H:%M:%S.%f", + ) + except ValueError as resp: + LOG.exception(f"Failed on {tc_trackfilename}") + raise (ValueError(f"FAILED ON {tc_trackfilename}: {resp}")) if file_timestamp < database_timestamp: LOG.info("") LOG.interactive( diff --git a/geoips/utils/memusg.py b/geoips/utils/memusg.py index 42f9eb606..942c01eed 100644 --- a/geoips/utils/memusg.py +++ b/geoips/utils/memusg.py @@ -3,6 +3,7 @@ """Utilities for tracking and monitoring memory and resource usage.""" # Python standard Libraries +# Python standard Libraries import logging import socket import os diff --git a/pyproject.toml b/pyproject.toml index 6da219a9c..609d77f4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,10 +56,12 @@ requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"] build-backend = "poetry_dynamic_versioning.backend" # object used to perform the build process [tool.poetry.dependencies] # must download to run +# NOTE this setuptools dependency should be removed altogether once meson fortran builds are working +setuptools = "<70" # Required for any fortran builds until meson working python = ">=3.10.0" # mandatory to declare the required python version matplotlib = ">=3.7.0" # Base requirement works, version specific to test outputs -netcdf4 = "*" # Base requirement -numpy = "<2.0.0" # Base requirement +netcdf4 = "<1.7.0" # Base requirement, netcdf 1.7.0 causes seg fault in ABI reader +numpy = "<2.0" # Base requirement numpy 1.26.4 still works, 2.0 breaks numexpr pyresample = "*" # Base requirement Geospatial image resampling # efficiency improvements >= 1.22.3 pyyaml = "*" # Base requirement @@ -78,12 +80,11 @@ pyaml-env = "*" # Reading YAML output config files, with paths pyorbital = "*" # required by satpy pypublicdecompwt = "*" # Required to read SEVIRI data rio-cogeo = "*" # Cloud Optimized GEOTIFF output -rasterio = "*" # GEOTIFF output -# version <1.3.10 required for bug in geotiff_standard -# Remove version requirement when bug fixed +rasterio = "*" # GEOTIFF output; Pinned to <1.3.10 due to the bug in geotiff_standard referencing = "*" satpy = "*" # efficiency improvements >= 0.33.1 scikit-image = "*" # Radius based center coverage checks +alphashape = ">=1.3.1" # For masking scipy's griddata output tabulate = "*" # Tables for the CLI colorama = "*" # Easy terminal color codes @@ -99,6 +100,12 @@ sphinxcontrib-autoprogram = { version = "*", optional = true } m2r2 = { version = "*", optional = true } brassy = { version = "*", optional = true } sphinx-argparse = { version = "*", optional = true } +# Version 3.0.2 causes error , 0.8.4 works +# Unsure why there was such a wide range of version numbers installed. +# File "lib/python3.10/site-packages/m2r2.py", line 82, in +# class RestBlockGrammar(mistune.BlockGrammar): +# AttributeError: module 'mistune' has no attribute 'BlockGrammar' +mistune = { version = "0.8.4", optional = true } # Lint group bandit = { version = "*", optional = true } black = { version = "*", optional = true } @@ -130,6 +137,7 @@ doc = [ "sphinxcontrib-autoprogram", # Required for adding command line options to documentation "m2r2", # Required for rendering markdown into RST-based documentation "brassy", # Release Note Generation + "mistune", # Required for rendering markdown into RST-based documentation ] lint = [ "bandit", # Syntax/security checking diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..f2e21dde9 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +addopts = -v -rf --ff --cov-report=term-missing +testpaths = + tests/unit_tests* +norecursedirs = xarray_utils \ No newline at end of file diff --git a/setup/check_system_requirements.sh b/setup/check_system_requirements.sh index fb81963af..6c4a154f5 100755 --- a/setup/check_system_requirements.sh +++ b/setup/check_system_requirements.sh @@ -63,6 +63,10 @@ if [[ "$1" == "geoips_full" ]]; then . $GEOIPS_PACKAGES_DIR/geoips/setup/check_system_requirements.sh openblas fi +if [[ "$1" == "set_gitconfig" ]]; then + source $GEOIPS_PACKAGES_DIR/geoips/setup/bash_setup/gitconfigs +fi + if [[ "$1" == "run_command" ]]; then echo "Running $2 ... " `$2 >> $install_log 2>&1` @@ -346,8 +350,11 @@ if [[ "$1" == "test_data" || "$1" == "test_data_github" ]]; then data_path="$test_data_dir/outputs/*" ls $data_path >> $install_log 2>&1 retval_outputs=$? + data_path="$test_data_dir/*.tgz" + ls $data_path >> $install_log 2>&1 + retval_tgz=$? - if [[ "$retval_data" != "0" && "$retval_docs" != "0" && "$retval_outputs" != "0" ]]; then + if [[ "$retval_data" != "0" && "$retval_docs" != "0" && "$retval_outputs" != "0" && "$retval_tgz" != "0" ]]; then if [[ "$exit_on_missing" == "true" ]]; then echo "FAILED: Missing $test_data_name_string" echo " Please run install script, then rerun test script. " @@ -378,6 +385,16 @@ if [[ "$1" == "test_data" || "$1" == "test_data_github" ]]; then echo "SUCCESS: successfully switch to branch $switch_to_branch" fi fi + if [[ -e $test_data_dir/uncompress_test_data.sh ]]; then + $test_data_dir/uncompress_test_data.sh >> $install_log 2>&1 + retval=$? + if [[ "$retval" == "0" ]]; then + echo "SUCCESS: Decompressed ${test_data_name}" + else + echo "FAILED: Failed to decompress ${test_data_name}. Try deleting and rerunning." + exit 1 + fi + fi else echo "DOWNLOADING: NextCloud Dataset $test_data_name @ $test_data_url" echo "python $SCRIPT_DIR/download_test_data.py $test_data_url $test_data_dir | tar -xz -C $GEOIPS_TESTDATA_DIR >> $install_log 2>&1" @@ -400,23 +417,12 @@ if [[ "$1" == "test_data" || "$1" == "test_data_github" ]]; then fi retval=$? if [[ "$retval" == "0" ]]; then - echo "SUCCESS: Decompressed ${test_data_name}" + echo "SUCCESS: Pulled ${test_data_name} from NexCloud ${test_data_url}" else - echo "FAILED: Failed to decompress ${test_data_name}" + echo "FAILED: Failed to pull ${test_data_name} from NexCloud ${test_data_url}" echo " try deleting and re-running" exit 1 fi - # If this is a github repo, then check if current-branch exists - # and switch to it if so. Allow branch not existing. - if [[ "$switch_to_branch" != "" ]]; then - echo "git -C $test_data_dir checkout $switch_to_branch >> $install_log 2>&1" - git -C $test_data_dir checkout $switch_to_branch >> $install_log 2>&1 - if [[ "$?" != "0" ]]; then - echo "Branch $switch_to_branch did not exist, staying on current branch" - else - echo "SUCCESS: successfully switch to branch $switch_to_branch" - fi - fi fi else echo "SUCCESS: $test_data_name_string appears to be installed successfully" diff --git a/setup/config_geoips b/setup/config_geoips index d05cf4a45..6b247dda9 100644 --- a/setup/config_geoips +++ b/setup/config_geoips @@ -39,7 +39,7 @@ export GEOIPS_PACKAGES_DIR=$GEOIPS_BASEDIR/geoips_packages export GEOIPS_TESTDATA_DIR=$GEOIPS_BASEDIR/test_data export CARTOPY_DATA_DIR=$GEOIPS_DEPENDENCIES_DIR/CARTOPY_DATA_DIR -export PYTHONPATH=$GEOIPS_BASEDIR/geoips_packages +# export PYTHONPATH=$GEOIPS_BASEDIR/geoips_packages export PATH=$GEOIPS_DEPENDENCIES_DIR/bin:$PATH ############################################################################ diff --git a/setup/geoips_conda_init_setup b/setup/geoips_conda_init_setup index f34392fcf..490ef1ab8 100644 --- a/setup/geoips_conda_init_setup +++ b/setup/geoips_conda_init_setup @@ -10,6 +10,7 @@ else GEOIPS_CONDA_DIR=$GEOIPS_DEPENDENCIES_DIR/miniconda3 fi +# DO NOT set PATH in here. Ever. if [[ -f $GEOIPS_CONDA_DIR/bin/conda ]]; then # >>> conda initialize >>> # !! Contents within this block are managed by 'conda init' !! @@ -19,21 +20,17 @@ if [[ -f $GEOIPS_CONDA_DIR/bin/conda ]]; then else if [ -f "$GEOIPS_CONDA_DIR/etc/profile.d/conda.sh" ]; then . "$GEOIPS_CONDA_DIR/etc/profile.d/conda.sh" - else - export PATH="$GEOIPS_DEPENDENCIES_DIR/miniconda3/bin:$PATH" + elif [ -f "$GEOIPS_DEPENDENCIES_DIR/miniconda3/etc/profile.d/mamba.sh" ]; then + . "$GEOIPS_DEPENDENCIES_DIR/miniconda3/etc/profile.d/mamba.sh" fi fi unset __conda_setup # <<< conda initialize <<< fi -export PATH="$GEOIPS_DEPENDENCIES_DIR/miniconda3/bin:$PATH" whichconda=`which conda 2> /dev/null` if [[ "$whichconda" != "" ]]; then if conda env list | grep -q --regex '^geoips_conda'; then eval "$(conda shell.bash activate geoips_conda)" fi fi -if [ -f "$GEOIPS_DEPENDENCIES_DIR/miniconda3/etc/profile.d/mamba.sh" ]; then - . "$GEOIPS_DEPENDENCIES_DIR/miniconda3/etc/profile.d/mamba.sh" -fi diff --git a/tests/outputs/abi.static.Infrared.imagery_annotated/20200918.195020.goes-16.abi.Infrared.goes_east.45p56.noaa.10p0.png b/tests/outputs/abi.static.Infrared.imagery_annotated/20200918.195020.goes-16.abi.Infrared.goes_east.45p56.noaa.10p0.png index 05f8da86f..8146a868d 100644 Binary files a/tests/outputs/abi.static.Infrared.imagery_annotated/20200918.195020.goes-16.abi.Infrared.goes_east.45p56.noaa.10p0.png and b/tests/outputs/abi.static.Infrared.imagery_annotated/20200918.195020.goes-16.abi.Infrared.goes_east.45p56.noaa.10p0.png differ diff --git a/tests/outputs/abi.static.Visible.imagery_annotated/20200918.195020.goes-16.abi.Visible.goes_east.41p12.noaa.10p0.png b/tests/outputs/abi.static.Visible.imagery_annotated/20200918.195020.goes-16.abi.Visible.goes_east.41p12.noaa.10p0.png index 0bbc7eb80..6fe68a766 100644 Binary files a/tests/outputs/abi.static.Visible.imagery_annotated/20200918.195020.goes-16.abi.Visible.goes_east.41p12.noaa.10p0.png and b/tests/outputs/abi.static.Visible.imagery_annotated/20200918.195020.goes-16.abi.Visible.goes_east.41p12.noaa.10p0.png differ diff --git a/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png b/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png index 5ac1ffad5..ce6686ae1 100644 Binary files a/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png and b/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png differ diff --git a/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png b/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png index f10181ffb..1e844d3f7 100644 Binary files a/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png and b/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png differ diff --git a/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png b/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png index 125a02e20..5a1ac7197 100644 Binary files a/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png and b/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png differ diff --git a/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png b/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png index 69b993814..0badc6ed9 100644 Binary files a/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png and b/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png differ diff --git a/tests/outputs/amsr2.tc.89H-Physical.imagery_annotated/20200518_073601_IO012020_amsr2_gcom-w1_89H-Physical_140kts_100p00_res1p0-cr300.png b/tests/outputs/amsr2.tc.89H-Physical.imagery_annotated/20200518_073601_IO012020_amsr2_gcom-w1_89H-Physical_140kts_100p00_res1p0-cr300.png index 25be9d23c..1957d803f 100644 Binary files a/tests/outputs/amsr2.tc.89H-Physical.imagery_annotated/20200518_073601_IO012020_amsr2_gcom-w1_89H-Physical_140kts_100p00_res1p0-cr300.png and b/tests/outputs/amsr2.tc.89H-Physical.imagery_annotated/20200518_073601_IO012020_amsr2_gcom-w1_89H-Physical_140kts_100p00_res1p0-cr300.png differ diff --git a/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgInfrared-Gray.png b/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgInfrared-Gray.png index a6bdabc33..e7004de57 100644 Binary files a/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgInfrared-Gray.png and b/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgInfrared-Gray.png differ diff --git a/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgVisible.png b/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgVisible.png index 273fac665..b357b1342 100644 Binary files a/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgVisible.png and b/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgVisible.png differ diff --git a/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgInfrared-Gray.png b/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgInfrared-Gray.png index c223173c0..f41d53cf1 100644 Binary files a/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgInfrared-Gray.png and b/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgInfrared-Gray.png differ diff --git a/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgVisible.png b/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgVisible.png index 31bbb8054..0e1c03478 100644 Binary files a/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgVisible.png and b/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgVisible.png differ diff --git a/tests/outputs/amsr2_ocean.tc.windspeed.imagery_clean/20200518_073601_IO012020_amsr2_gcom-w1_windspeed_140kts_85p45_1p0-clean.png b/tests/outputs/amsr2_ocean.tc.windspeed.imagery_clean/20200518_073601_IO012020_amsr2_gcom-w1_windspeed_140kts_85p45_1p0-clean.png index ac69b0765..189e4a16d 100644 Binary files a/tests/outputs/amsr2_ocean.tc.windspeed.imagery_clean/20200518_073601_IO012020_amsr2_gcom-w1_windspeed_140kts_85p45_1p0-clean.png and b/tests/outputs/amsr2_ocean.tc.windspeed.imagery_clean/20200518_073601_IO012020_amsr2_gcom-w1_windspeed_140kts_85p45_1p0-clean.png differ diff --git a/tests/outputs/ascat_knmi.tc.windbarbs.imagery_windbarbs_clean/20210421_014248_WP022021_ascat_metop-c_windbarbs_120kts_78p20_0p5-clean.png b/tests/outputs/ascat_knmi.tc.windbarbs.imagery_windbarbs_clean/20210421_014248_WP022021_ascat_metop-c_windbarbs_120kts_78p20_0p5-clean.png index 93a5fd3cd..83a0876f2 100644 Binary files a/tests/outputs/ascat_knmi.tc.windbarbs.imagery_windbarbs_clean/20210421_014248_WP022021_ascat_metop-c_windbarbs_120kts_78p20_0p5-clean.png and b/tests/outputs/ascat_knmi.tc.windbarbs.imagery_windbarbs_clean/20210421_014248_WP022021_ascat_metop-c_windbarbs_120kts_78p20_0p5-clean.png differ diff --git a/tests/outputs/ascat_low_knmi.tc.windbarbs.imagery_windbarbs/20210421_014156_WP022021_ascat_metop-c_windbarbs_120kts_35p17_1p0.png b/tests/outputs/ascat_low_knmi.tc.windbarbs.imagery_windbarbs/20210421_014156_WP022021_ascat_metop-c_windbarbs_120kts_35p17_1p0.png index e8f09633d..77ce8d059 100644 Binary files a/tests/outputs/ascat_low_knmi.tc.windbarbs.imagery_windbarbs/20210421_014156_WP022021_ascat_metop-c_windbarbs_120kts_35p17_1p0.png and b/tests/outputs/ascat_low_knmi.tc.windbarbs.imagery_windbarbs/20210421_014156_WP022021_ascat_metop-c_windbarbs_120kts_35p17_1p0.png differ diff --git a/tests/outputs/ascat_noaa_25km.tc.windbarbs.imagery_windbarbs/20230524_235304_WP022023_ascat_metop-c_windbarbs_135kts_39p90_0p7.png b/tests/outputs/ascat_noaa_25km.tc.windbarbs.imagery_windbarbs/20230524_235304_WP022023_ascat_metop-c_windbarbs_135kts_39p90_0p7.png index 3ad2822b4..9977854a9 100644 Binary files a/tests/outputs/ascat_noaa_25km.tc.windbarbs.imagery_windbarbs/20230524_235304_WP022023_ascat_metop-c_windbarbs_135kts_39p90_0p7.png and b/tests/outputs/ascat_noaa_25km.tc.windbarbs.imagery_windbarbs/20230524_235304_WP022023_ascat_metop-c_windbarbs_135kts_39p90_0p7.png differ diff --git a/tests/outputs/ascat_noaa_50km.tc.wind-ambiguities.imagery_windbarbs/20230524_235200_WP022023_ascat_metop-c_wind-ambiguities_135kts_50p08_1p1.png b/tests/outputs/ascat_noaa_50km.tc.wind-ambiguities.imagery_windbarbs/20230524_235200_WP022023_ascat_metop-c_wind-ambiguities_135kts_50p08_1p1.png index 06b592556..4e897b214 100644 Binary files a/tests/outputs/ascat_noaa_50km.tc.wind-ambiguities.imagery_windbarbs/20230524_235200_WP022023_ascat_metop-c_wind-ambiguities_135kts_50p08_1p1.png and b/tests/outputs/ascat_noaa_50km.tc.wind-ambiguities.imagery_windbarbs/20230524_235200_WP022023_ascat_metop-c_wind-ambiguities_135kts_50p08_1p1.png differ diff --git a/tests/outputs/ascat_uhr.tc.wind-ambiguities.imagery_windbarbs/20210421_014200_WP022021_ascatuhr_metop-c_wind-ambiguities_120kts_100p00_0p1.png b/tests/outputs/ascat_uhr.tc.wind-ambiguities.imagery_windbarbs/20210421_014200_WP022021_ascatuhr_metop-c_wind-ambiguities_120kts_100p00_0p1.png index f9868dc95..4ab800173 100644 Binary files a/tests/outputs/ascat_uhr.tc.wind-ambiguities.imagery_windbarbs/20210421_014200_WP022021_ascatuhr_metop-c_wind-ambiguities_120kts_100p00_0p1.png and b/tests/outputs/ascat_uhr.tc.wind-ambiguities.imagery_windbarbs/20210421_014200_WP022021_ascatuhr_metop-c_wind-ambiguities_120kts_100p00_0p1.png differ diff --git a/tests/outputs/ascat_uhr.tc.windbarbs.imagery_windbarbs/20230722_234513_AL052023_ascatuhr_metop-b_windbarbs_65kts_63p79_1p0.png b/tests/outputs/ascat_uhr.tc.windbarbs.imagery_windbarbs/20230722_234513_AL052023_ascatuhr_metop-b_windbarbs_65kts_63p79_1p0.png index 85ba01a6d..7e6715ff1 100644 Binary files a/tests/outputs/ascat_uhr.tc.windbarbs.imagery_windbarbs/20230722_234513_AL052023_ascatuhr_metop-b_windbarbs_65kts_63p79_1p0.png and b/tests/outputs/ascat_uhr.tc.windbarbs.imagery_windbarbs/20230722_234513_AL052023_ascatuhr_metop-b_windbarbs_65kts_63p79_1p0.png differ diff --git a/tests/outputs/ascat_uhr.tc.windspeed.imagery_clean/20230912_011802_AL132023_ascatuhr_metop-c_windspeed_100kts_51p44_1p0.png b/tests/outputs/ascat_uhr.tc.windspeed.imagery_clean/20230912_011802_AL132023_ascatuhr_metop-c_windspeed_100kts_51p44_1p0.png index 82ef9a68e..c1ab41c6a 100644 Binary files a/tests/outputs/ascat_uhr.tc.windspeed.imagery_clean/20230912_011802_AL132023_ascatuhr_metop-c_windspeed_100kts_51p44_1p0.png and b/tests/outputs/ascat_uhr.tc.windspeed.imagery_clean/20230912_011802_AL132023_ascatuhr_metop-c_windspeed_100kts_51p44_1p0.png differ diff --git a/tests/outputs/cygnss.tc.windspeed.imagery_clean/20240125_060307_SH062024_cygnss_cygnss_windspeed_80kts_6p22_1p0-clean.png b/tests/outputs/cygnss.tc.windspeed.imagery_clean/20240125_060307_SH062024_cygnss_cygnss_windspeed_80kts_6p22_1p0-clean.png new file mode 100644 index 000000000..371c272b9 Binary files /dev/null and b/tests/outputs/cygnss.tc.windspeed.imagery_clean/20240125_060307_SH062024_cygnss_cygnss_windspeed_80kts_6p22_1p0-clean.png differ diff --git a/tests/outputs/hy2.tc.windspeed.imagery_annotated/20211202_084039_WP272021_hscat_hy-2b_windspeed_95kts_97p06_1p0.png b/tests/outputs/hy2.tc.windspeed.imagery_annotated/20211202_084039_WP272021_hscat_hy-2b_windspeed_95kts_97p06_1p0.png index 37d84155d..5fdfd3dfb 100644 Binary files a/tests/outputs/hy2.tc.windspeed.imagery_annotated/20211202_084039_WP272021_hscat_hy-2b_windspeed_95kts_97p06_1p0.png and b/tests/outputs/hy2.tc.windspeed.imagery_annotated/20211202_084039_WP272021_hscat_hy-2b_windspeed_95kts_97p06_1p0.png differ diff --git a/tests/outputs/oscat_knmi.tc.windbarbs.imagery_windbarbs/20210209_025351_SH192021_oscat_scatsat-1_windbarbs_135kts_75p10_1p0.png b/tests/outputs/oscat_knmi.tc.windbarbs.imagery_windbarbs/20210209_025351_SH192021_oscat_scatsat-1_windbarbs_135kts_75p10_1p0.png index a4dfad843..2d5d736cc 100644 Binary files a/tests/outputs/oscat_knmi.tc.windbarbs.imagery_windbarbs/20210209_025351_SH192021_oscat_scatsat-1_windbarbs_135kts_75p10_1p0.png and b/tests/outputs/oscat_knmi.tc.windbarbs.imagery_windbarbs/20210209_025351_SH192021_oscat_scatsat-1_windbarbs_135kts_75p10_1p0.png differ diff --git a/tests/outputs/smap.tc.windspeed.imagery_clean/20210926_210400_WP202021_smap-spd_smap_windspeed_100kts_74p87_1p0-clean.png b/tests/outputs/smap.tc.windspeed.imagery_clean/20210926_210400_WP202021_smap-spd_smap_windspeed_100kts_74p87_1p0-clean.png index 8d1b8e4da..5322afa7e 100644 Binary files a/tests/outputs/smap.tc.windspeed.imagery_clean/20210926_210400_WP202021_smap-spd_smap_windspeed_100kts_74p87_1p0-clean.png and b/tests/outputs/smap.tc.windspeed.imagery_clean/20210926_210400_WP202021_smap-spd_smap_windspeed_100kts_74p87_1p0-clean.png differ diff --git a/tests/scripts/atms.tc.165H.netcdf_geoips.sh b/tests/scripts/atms.tc.165H.netcdf_geoips.sh index 4d8048d72..5543ebf35 100755 --- a/tests/scripts/atms.tc.165H.netcdf_geoips.sh +++ b/tests/scripts/atms.tc.165H.netcdf_geoips.sh @@ -31,7 +31,7 @@ geoips run single_source \ --compare_path "$GEOIPS_PACKAGES_DIR/geoips/tests/outputs/atms.tc..netcdf_geoips" \ --tc_spec_template tc_4km_256x256 \ --product_spec_override '{}' \ - --output_formatter_kwargs '{}' \ + --output_formatter_kwargs '{"clobber": "True"}' \ --filename_formatter_kwargs '{}' \ --metadata_output_formatter_kwargs '{}' \ --metadata_filename_formatter_kwargs '{}' diff --git a/tests/scripts/cygnss.tc.windspeed.imagery_clean.sh b/tests/scripts/cygnss.tc.windspeed.imagery_clean.sh new file mode 100755 index 000000000..8e4b3b030 --- /dev/null +++ b/tests/scripts/cygnss.tc.windspeed.imagery_clean.sh @@ -0,0 +1,19 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + +geoips run single_source \ + $GEOIPS_TESTDATA_DIR/test_data_cygnss/data/cyg.ddmi.s20240125-000001-e20240125-235959.l2.wind_trackgridsize25km_NOAAv1.2_L1a21.d21.nc \ + --reader_name cygnss_netcdf \ + --product_name windspeed \ + --output_formatter imagery_clean \ + --filename_formatter tc_clean_fname \ + --window_start_time "20240125T0600Z" \ + --window_end_time "20240125T0615Z" \ + --minimum_coverage 0 \ + --trackfile_parser bdeck_parser \ + --trackfiles $GEOIPS_PACKAGES_DIR/geoips/tests/sectors/tc_bdecks/bsh062024.dat \ + --compare_path "$GEOIPS_PACKAGES_DIR/geoips/tests/outputs/cygnss.tc.windspeed.imagery_clean" \ + +ss_retval=$? + +exit $((ss_retval)) \ No newline at end of file diff --git a/tests/sectors/tc_bdecks/bio012024.dat b/tests/sectors/tc_bdecks/bio012024.dat new file mode 100644 index 000000000..e81f6ccf7 --- /dev/null +++ b/tests/sectors/tc_bdecks/bio012024.dat @@ -0,0 +1,25 @@ +IO, 01, 2024052312, , BEST, 0, 136N, 866E, 20, 1001, DB, 0, , 0, 0, 0, 0, 1003, 150, 0, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052318, , BEST, 0, 143N, 876E, 20, 999, DB, 0, , 0, 0, 0, 0, 1002, 280, 0, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052400, , BEST, 0, 149N, 884E, 20, 997, DB, 0, , 0, 0, 0, 0, 999, 185, 110, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052406, , BEST, 0, 157N, 891E, 25, 996, TD, 0, , 0, 0, 0, 0, 1000, 205, 100, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052412, , BEST, 0, 164N, 894E, 25, 996, TD, 0, , 0, 0, 0, 0, 1000, 205, 90, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052418, , BEST, 0, 170N, 895E, 25, 996, TD, 0, , 0, 0, 0, 0, 1002, 220, 90, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052500, , BEST, 0, 177N, 894E, 25, 993, TD, 0, , 0, 0, 0, 0, 1000, 220, 80, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052506, , BEST, 0, 180N, 893E, 30, 993, TD, 0, , 0, 0, 0, 0, 998, 220, 80, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052512, , BEST, 0, 183N, 892E, 35, 990, TS, 34, NEQ, 105, 130, 150, 0, 998, 220, 70, 0, 0, B, 0, , 0, 0, ONE, S, 0, , 0, 0, 0, 0, TRANSITIONED, ioA92024 to io012024, +IO, 01, 2024052518, , BEST, 0, 189N, 891E, 35, 990, TS, 34, NEQ, 115, 125, 0, 0, 1000, 220, 75, 0, 0, B, 0, , 0, 0, REMAL, M, +IO, 01, 2024052600, , BEST, 0, 195N, 890E, 45, 990, TS, 34, NEQ, 115, 125, 0, 0, 1000, 220, 75, 0, 0, B, 0, , 0, 0, REMAL, M, +IO, 01, 2024052606, , BEST, 0, 202N, 888E, 55, 985, TS, 34, NEQ, 165, 130, 130, 95, 998, 190, 65, 0, 0, B, 0, , 0, 0, REMAL, M, +IO, 01, 2024052606, , BEST, 0, 202N, 888E, 55, 985, TS, 50, NEQ, 80, 90, 90, 0, 998, 190, 65, 0, 0, B, 0, , 0, 0, REMAL, M, +IO, 01, 2024052612, , BEST, 0, 209N, 887E, 55, 980, TS, 34, NEQ, 145, 130, 130, 105, 998, 180, 55, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052612, , BEST, 0, 209N, 887E, 55, 980, TS, 50, NEQ, 70, 75, 80, 30, 998, 180, 55, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052618, , BEST, 0, 218N, 886E, 55, 978, TS, 34, NEQ, 145, 130, 130, 105, 996, 160, 50, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052618, , BEST, 0, 218N, 886E, 55, 978, TS, 50, NEQ, 70, 75, 80, 30, 996, 160, 50, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052700, , BEST, 0, 226N, 889E, 50, 979, TS, 34, NEQ, 145, 130, 130, 105, 995, 200, 50, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052700, , BEST, 0, 226N, 889E, 50, 979, TS, 50, NEQ, 70, 75, 80, 30, 995, 200, 50, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052706, , BEST, 0, 232N, 893E, 45, 981, TS, 34, NEQ, 120, 135, 120, 85, 995, 170, 60, 0, 0, B, 0, , 0, 0, REMAL, , +IO, 01, 2024052712, , BEST, 0, 237N, 899E, 40, 983, TS, 34, NEQ, 105, 165, 135, 85, 995, 155, 65, 0, 0, B, 0, , 0, 0, REMAL, , +IO, 01, 2024052718, , BEST, 0, 242N, 904E, 40, 984, TS, 34, NEQ, 110, 170, 140, 90, 996, 130, 65, 0, 0, B, 0, , 0, 0, REMAL, M, +IO, 01, 2024052800, , BEST, 0, 249N, 912E, 40, 985, TS, 34, NEQ, 115, 175, 145, 95, 995, 105, 70, 0, 0, B, 0, , 0, 0, REMAL, , +IO, 01, 2024052806, , BEST, 0, 254N, 919E, 30, 990, TD, 0, , 0, 0, 0, 0, 998, 95, 60, 0, 0, B, 0, , 0, 0, REMAL, , +IO, 01, 2024052812, , BEST, 0, 263N, 931E, 30, 990, TD, 0, , 0, 0, 0, 0, 998, 95, 60, 0, 0, B, 0, , 0, 0, REMAL, , diff --git a/tests/sectors/tc_bdecks/bsh062024.dat b/tests/sectors/tc_bdecks/bsh062024.dat new file mode 100644 index 000000000..455c713c5 --- /dev/null +++ b/tests/sectors/tc_bdecks/bsh062024.dat @@ -0,0 +1,168 @@ +SH, 06, 2024011012, , BEST, 0, 59S, 868E, 15, 1008, DB, 0, , 0, 0, 0, 0, 1009, 155, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011018, , BEST, 0, 62S, 880E, 15, 1008, DB, 0, , 0, 0, 0, 0, 1009, 155, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011100, , BEST, 0, 65S, 891E, 15, 1008, DB, 0, , 0, 0, 0, 0, 1009, 155, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011106, , BEST, 0, 71S, 899E, 15, 1008, DB, 0, , 0, 0, 0, 0, 1009, 155, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011112, , BEST, 0, 76S, 905E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1009, 255, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011118, , BEST, 0, 81S, 911E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1009, 255, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011200, , BEST, 0, 87S, 917E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1009, 255, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011206, , BEST, 0, 95S, 925E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1009, 280, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011212, , BEST, 0, 100S, 929E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1008, 200, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011218, , BEST, 0, 105S, 932E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1008, 200, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011300, , BEST, 0, 108S, 934E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1008, 200, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011306, , BEST, 0, 112S, 936E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1008, 200, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011312, , BEST, 0, 115S, 938E, 25, 1006, TD, 0, , 0, 0, 0, 0, 1008, 210, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011318, , BEST, 0, 116S, 940E, 25, 1005, TD, 0, , 0, 0, 0, 0, 1009, 210, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011400, , BEST, 0, 112S, 940E, 25, 1006, TD, 0, , 0, 0, 0, 0, 1008, 210, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011406, , BEST, 0, 108S, 938E, 25, 1005, TD, 0, , 0, 0, 0, 0, 1008, 180, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011412, , BEST, 0, 106S, 938E, 25, 1005, TD, 0, , 0, 0, 0, 0, 1008, 180, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011418, , BEST, 0, 104S, 937E, 30, 1005, TD, 0, , 0, 0, 0, 0, 1008, 180, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011500, , BEST, 0, 100S, 936E, 30, 1004, TD, 0, , 0, 0, 0, 0, 1008, 180, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011506, , BEST, 0, 96S, 936E, 35, 1001, TS, 34, NEQ, 20, 50, 50, 75, 1008, 170, 35, 0, 0, S, 0, , 0, 0, INVEST, M, 0, , 0, 0, 0, 0, TRANSITIONED, shA82024 to sh062024, +SH, 06, 2024011512, , BEST, 0, 95S, 937E, 40, 1001, TS, 34, NEQ, 65, 55, 50, 55, 1008, 170, 35, 0, 0, S, 0, , 0, 0, SIX, M, +SH, 06, 2024011518, , BEST, 0, 96S, 938E, 40, 998, TS, 34, NEQ, 65, 45, 50, 65, 1008, 155, 25, 0, 0, S, 0, , 0, 0, SIX, M, +SH, 06, 2024011600, , BEST, 0, 95S, 939E, 40, 998, TS, 34, NEQ, 65, 45, 50, 65, 1008, 180, 25, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011606, , BEST, 0, 95S, 938E, 45, 995, TS, 34, NEQ, 65, 45, 50, 65, 1008, 200, 15, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011612, , BEST, 0, 96S, 940E, 45, 994, TS, 34, NEQ, 65, 45, 50, 65, 1007, 230, 15, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011618, , BEST, 0, 96S, 942E, 45, 994, TS, 34, NEQ, 65, 45, 50, 65, 1007, 230, 15, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011700, , BEST, 0, 98S, 941E, 50, 997, TS, 34, NEQ, 65, 45, 50, 65, 1007, 250, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011706, , BEST, 0, 101S, 940E, 50, 997, TS, 34, NEQ, 65, 45, 50, 65, 1007, 250, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011706, , BEST, 0, 101S, 940E, 50, 997, TS, 50, NEQ, 40, 0, 0, 0, 1007, 250, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011712, , BEST, 0, 104S, 940E, 45, 997, TS, 34, NEQ, 65, 45, 50, 65, 1008, 320, 50, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011718, , BEST, 0, 109S, 940E, 45, 996, TS, 34, NEQ, 65, 40, 45, 65, 1009, 340, 55, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011800, , BEST, 0, 112S, 939E, 45, 989, TS, 34, NEQ, 70, 70, 65, 80, 1008, 210, 55, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011806, , BEST, 0, 116S, 938E, 45, 997, TS, 34, NEQ, 65, 40, 45, 65, 1008, 220, 35, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011812, , BEST, 0, 119S, 937E, 45, 997, TS, 34, NEQ, 65, 40, 45, 65, 1008, 220, 35, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011818, , BEST, 0, 122S, 935E, 45, 993, TS, 34, NEQ, 70, 70, 65, 95, 1008, 225, 75, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011900, , BEST, 0, 124S, 933E, 55, 989, TS, 34, NEQ, 75, 115, 100, 85, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011900, , BEST, 0, 124S, 933E, 55, 989, TS, 50, NEQ, 30, 55, 50, 40, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011906, , BEST, 0, 125S, 930E, 60, 988, TS, 34, NEQ, 75, 115, 100, 85, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011906, , BEST, 0, 125S, 930E, 60, 988, TS, 50, NEQ, 30, 55, 50, 40, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011912, , BEST, 0, 126S, 927E, 60, 988, TS, 34, NEQ, 75, 115, 100, 85, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011912, , BEST, 0, 126S, 927E, 60, 988, TS, 50, NEQ, 30, 55, 50, 40, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011918, , BEST, 0, 126S, 925E, 70, 988, TY, 34, NEQ, 90, 100, 95, 110, 1008, 180, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011918, , BEST, 0, 126S, 925E, 70, 988, TY, 50, NEQ, 50, 45, 45, 50, 1008, 180, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011918, , BEST, 0, 126S, 925E, 70, 988, TY, 64, NEQ, 30, 30, 25, 25, 1008, 180, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012000, , BEST, 0, 124S, 928E, 75, 988, TY, 34, NEQ, 85, 75, 95, 115, 1008, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012000, , BEST, 0, 124S, 928E, 75, 988, TY, 50, NEQ, 55, 45, 45, 50, 1008, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012000, , BEST, 0, 124S, 928E, 75, 988, TY, 64, NEQ, 30, 30, 25, 25, 1008, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012006, , BEST, 0, 125S, 926E, 70, 988, TY, 34, NEQ, 85, 75, 95, 115, 1009, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012006, , BEST, 0, 125S, 926E, 70, 988, TY, 50, NEQ, 55, 45, 45, 50, 1009, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012006, , BEST, 0, 125S, 926E, 70, 988, TY, 64, NEQ, 30, 30, 25, 25, 1009, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012012, , BEST, 0, 124S, 925E, 70, 980, TY, 34, NEQ, 75, 75, 80, 70, 1008, 230, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012012, , BEST, 0, 124S, 925E, 70, 980, TY, 50, NEQ, 40, 40, 40, 40, 1008, 230, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012012, , BEST, 0, 124S, 925E, 70, 980, TY, 64, NEQ, 25, 25, 25, 25, 1008, 230, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012018, , BEST, 0, 123S, 924E, 70, 981, TY, 34, NEQ, 75, 75, 80, 70, 1009, 200, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012018, , BEST, 0, 123S, 924E, 70, 981, TY, 50, NEQ, 40, 40, 40, 40, 1009, 200, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012018, , BEST, 0, 123S, 924E, 70, 981, TY, 64, NEQ, 25, 25, 25, 25, 1009, 200, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012100, , BEST, 0, 121S, 924E, 70, 984, TY, 34, NEQ, 65, 85, 85, 100, 1008, 150, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012100, , BEST, 0, 121S, 924E, 70, 984, TY, 50, NEQ, 40, 40, 40, 30, 1008, 150, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012100, , BEST, 0, 121S, 924E, 70, 984, TY, 64, NEQ, 25, 35, 30, 20, 1008, 150, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012106, , BEST, 0, 120S, 922E, 65, 984, TY, 34, NEQ, 65, 85, 85, 100, 1010, 225, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012106, , BEST, 0, 120S, 922E, 65, 984, TY, 50, NEQ, 40, 40, 40, 30, 1010, 225, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012106, , BEST, 0, 120S, 922E, 65, 984, TY, 64, NEQ, 25, 35, 30, 20, 1010, 225, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012112, , BEST, 0, 122S, 920E, 60, 985, TS, 34, NEQ, 80, 60, 80, 135, 1009, 265, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012112, , BEST, 0, 122S, 920E, 60, 985, TS, 50, NEQ, 35, 35, 35, 70, 1009, 265, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012118, , BEST, 0, 123S, 918E, 50, 994, TS, 34, NEQ, 80, 60, 80, 135, 1009, 180, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012118, , BEST, 0, 123S, 918E, 50, 994, TS, 50, NEQ, 35, 35, 35, 70, 1009, 180, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012200, , BEST, 0, 125S, 917E, 45, 991, TS, 34, NEQ, 80, 60, 80, 135, 1009, 170, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012206, , BEST, 0, 123S, 915E, 45, 999, TS, 34, NEQ, 80, 60, 80, 135, 1011, 270, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012212, , BEST, 0, 124S, 914E, 40, 1000, TS, 34, NEQ, 80, 60, 80, 135, 1010, 250, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012218, , BEST, 0, 126S, 915E, 40, 1000, TS, 34, NEQ, 50, 60, 50, 30, 1010, 250, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012300, , BEST, 0, 128S, 916E, 40, 1000, TS, 34, NEQ, 50, 60, 50, 30, 1010, 200, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012306, , BEST, 0, 130S, 916E, 45, 998, TS, 34, NEQ, 50, 50, 35, 0, 1010, 200, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012312, , BEST, 0, 133S, 916E, 50, 995, TS, 34, NEQ, 50, 50, 35, 0, 1010, 200, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012318, , BEST, 0, 136S, 915E, 50, 994, TS, 34, NEQ, 50, 65, 60, 40, 1009, 180, 25, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012318, , BEST, 0, 136S, 915E, 50, 994, TS, 50, NEQ, 30, 30, 25, 15, 1009, 180, 25, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012400, , BEST, 0, 139S, 913E, 55, 993, TS, 34, NEQ, 60, 75, 70, 45, 1010, 165, 25, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012400, , BEST, 0, 139S, 913E, 55, 993, TS, 50, NEQ, 30, 35, 25, 15, 1010, 165, 25, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012406, , BEST, 0, 140S, 912E, 65, 987, TY, 34, NEQ, 60, 75, 70, 45, 1010, 165, 15, 0, 20, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012406, , BEST, 0, 140S, 912E, 65, 987, TY, 50, NEQ, 30, 35, 25, 25, 1010, 165, 15, 0, 20, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012406, , BEST, 0, 140S, 912E, 65, 987, TY, 64, NEQ, 15, 15, 15, 15, 1010, 165, 15, 0, 20, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012412, , BEST, 0, 142S, 911E, 75, 980, TY, 34, NEQ, 60, 75, 70, 45, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012412, , BEST, 0, 142S, 911E, 75, 980, TY, 50, NEQ, 30, 35, 25, 25, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012412, , BEST, 0, 142S, 911E, 75, 980, TY, 64, NEQ, 15, 15, 15, 15, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012418, , BEST, 0, 145S, 911E, 80, 976, TY, 34, NEQ, 60, 75, 70, 45, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012418, , BEST, 0, 145S, 911E, 80, 976, TY, 50, NEQ, 30, 35, 25, 25, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012418, , BEST, 0, 145S, 911E, 80, 976, TY, 64, NEQ, 15, 15, 15, 15, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012500, , BEST, 0, 147S, 908E, 80, 975, TY, 34, NEQ, 60, 75, 70, 45, 1009, 140, 15, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012500, , BEST, 0, 147S, 908E, 80, 975, TY, 50, NEQ, 30, 35, 25, 25, 1009, 140, 15, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012500, , BEST, 0, 147S, 908E, 80, 975, TY, 64, NEQ, 15, 15, 15, 15, 1009, 140, 15, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012506, , BEST, 0, 151S, 900E, 80, 978, TY, 34, NEQ, 60, 75, 125, 85, 1011, 170, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012506, , BEST, 0, 151S, 900E, 80, 978, TY, 50, NEQ, 40, 50, 45, 40, 1011, 170, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012506, , BEST, 0, 151S, 900E, 80, 978, TY, 64, NEQ, 30, 35, 30, 25, 1011, 170, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012512, , BEST, 0, 155S, 893E, 90, 974, TY, 34, NEQ, 70, 130, 115, 50, 1010, 150, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012512, , BEST, 0, 155S, 893E, 90, 974, TY, 50, NEQ, 35, 60, 55, 35, 1010, 150, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012512, , BEST, 0, 155S, 893E, 90, 974, TY, 64, NEQ, 25, 40, 40, 25, 1010, 150, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012518, , BEST, 0, 160S, 883E, 95, 972, TY, 34, NEQ, 70, 130, 115, 50, 1011, 120, 20, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012518, , BEST, 0, 160S, 883E, 95, 972, TY, 50, NEQ, 35, 60, 55, 35, 1011, 120, 20, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012518, , BEST, 0, 160S, 883E, 95, 972, TY, 64, NEQ, 25, 40, 40, 30, 1011, 120, 20, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012600, , BEST, 0, 163S, 875E, 105, 958, TY, 34, NEQ, 60, 130, 110, 55, 1009, 105, 15, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012600, , BEST, 0, 163S, 875E, 105, 958, TY, 50, NEQ, 35, 50, 45, 35, 1009, 105, 15, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012600, , BEST, 0, 163S, 875E, 105, 958, TY, 64, NEQ, 30, 30, 30, 25, 1009, 105, 15, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012606, , BEST, 0, 169S, 865E, 105, 958, TY, 34, NEQ, 60, 130, 110, 55, 1011, 150, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012606, , BEST, 0, 169S, 865E, 105, 958, TY, 50, NEQ, 35, 50, 45, 35, 1011, 150, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012606, , BEST, 0, 169S, 865E, 105, 958, TY, 64, NEQ, 30, 30, 30, 25, 1011, 150, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012612, , BEST, 0, 175S, 853E, 110, 958, TY, 34, NEQ, 65, 95, 90, 45, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012612, , BEST, 0, 175S, 853E, 110, 958, TY, 50, NEQ, 40, 45, 40, 30, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012612, , BEST, 0, 175S, 853E, 110, 958, TY, 64, NEQ, 30, 35, 30, 20, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012618, , BEST, 0, 182S, 842E, 105, 960, TY, 34, NEQ, 65, 95, 90, 45, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012618, , BEST, 0, 182S, 842E, 105, 960, TY, 50, NEQ, 40, 45, 40, 30, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012618, , BEST, 0, 182S, 842E, 105, 960, TY, 64, NEQ, 30, 35, 30, 20, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012700, , BEST, 0, 187S, 831E, 100, 964, TY, 34, NEQ, 65, 95, 90, 45, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012700, , BEST, 0, 187S, 831E, 100, 964, TY, 50, NEQ, 40, 45, 40, 30, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012700, , BEST, 0, 187S, 831E, 100, 964, TY, 64, NEQ, 30, 35, 30, 20, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012706, , BEST, 0, 190S, 820E, 100, 964, TY, 34, NEQ, 45, 55, 50, 50, 1010, 140, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012706, , BEST, 0, 190S, 820E, 100, 964, TY, 50, NEQ, 30, 30, 35, 25, 1010, 140, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012706, , BEST, 0, 190S, 820E, 100, 964, TY, 64, NEQ, 25, 25, 30, 20, 1010, 140, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012712, , BEST, 0, 193S, 806E, 100, 964, TY, 34, NEQ, 45, 55, 50, 50, 1010, 160, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012712, , BEST, 0, 193S, 806E, 100, 964, TY, 50, NEQ, 30, 30, 35, 25, 1010, 160, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012712, , BEST, 0, 193S, 806E, 100, 964, TY, 64, NEQ, 25, 25, 30, 20, 1010, 160, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012718, , BEST, 0, 195S, 794E, 100, 965, TY, 34, NEQ, 55, 80, 80, 55, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012718, , BEST, 0, 195S, 794E, 100, 965, TY, 50, NEQ, 20, 40, 40, 20, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012718, , BEST, 0, 195S, 794E, 100, 965, TY, 64, NEQ, 10, 20, 20, 10, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012800, , BEST, 0, 200S, 779E, 95, 968, TY, 34, NEQ, 55, 80, 80, 55, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012800, , BEST, 0, 200S, 779E, 95, 968, TY, 50, NEQ, 20, 40, 40, 20, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012800, , BEST, 0, 200S, 779E, 95, 968, TY, 64, NEQ, 10, 20, 20, 10, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012806, , BEST, 0, 204S, 764E, 95, 970, TY, 34, NEQ, 55, 90, 60, 45, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012806, , BEST, 0, 204S, 764E, 95, 970, TY, 50, NEQ, 35, 35, 30, 20, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012806, , BEST, 0, 204S, 764E, 95, 970, TY, 64, NEQ, 20, 20, 20, 15, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012812, , BEST, 0, 210S, 748E, 100, 970, TY, 34, NEQ, 60, 85, 65, 45, 1008, 105, 10, 0, 10, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012812, , BEST, 0, 210S, 748E, 100, 970, TY, 50, NEQ, 30, 30, 30, 30, 1008, 105, 10, 0, 10, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012812, , BEST, 0, 210S, 748E, 100, 970, TY, 64, NEQ, 20, 20, 20, 20, 1008, 105, 10, 0, 10, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012818, , BEST, 0, 217S, 732E, 110, 955, TY, 34, NEQ, 60, 85, 70, 45, 1010, 110, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012818, , BEST, 0, 217S, 732E, 110, 955, TY, 50, NEQ, 40, 40, 30, 30, 1010, 110, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012818, , BEST, 0, 217S, 732E, 110, 955, TY, 64, NEQ, 20, 20, 15, 15, 1010, 110, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012900, , BEST, 0, 227S, 719E, 115, 954, TY, 34, NEQ, 60, 75, 60, 60, 1010, 115, 10, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012900, , BEST, 0, 227S, 719E, 115, 954, TY, 50, NEQ, 40, 45, 45, 35, 1010, 115, 10, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012900, , BEST, 0, 227S, 719E, 115, 954, TY, 64, NEQ, 30, 40, 35, 30, 1010, 115, 10, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012906, , BEST, 0, 238S, 711E, 115, 948, TY, 34, NEQ, 60, 75, 60, 60, 1010, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012906, , BEST, 0, 238S, 711E, 115, 948, TY, 50, NEQ, 40, 45, 45, 35, 1010, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012906, , BEST, 0, 238S, 711E, 115, 948, TY, 64, NEQ, 30, 40, 35, 30, 1010, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012912, , BEST, 0, 251S, 708E, 120, 960, TY, 34, NEQ, 60, 75, 60, 60, 1009, 160, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012912, , BEST, 0, 251S, 708E, 120, 960, TY, 50, NEQ, 40, 45, 45, 35, 1009, 160, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012912, , BEST, 0, 251S, 708E, 120, 960, TY, 64, NEQ, 30, 40, 35, 30, 1009, 160, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012918, , BEST, 0, 264S, 709E, 110, 950, TY, 34, NEQ, 60, 75, 60, 60, 1010, 125, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012918, , BEST, 0, 264S, 709E, 110, 950, TY, 50, NEQ, 40, 45, 45, 35, 1010, 125, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012918, , BEST, 0, 264S, 709E, 110, 950, TY, 64, NEQ, 30, 40, 35, 30, 1010, 125, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013000, , BEST, 0, 280S, 717E, 105, 952, TY, 34, NEQ, 65, 65, 65, 65, 1009, 130, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013000, , BEST, 0, 280S, 717E, 105, 952, TY, 50, NEQ, 40, 45, 45, 40, 1009, 130, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013000, , BEST, 0, 280S, 717E, 105, 952, TY, 64, NEQ, 35, 40, 35, 35, 1009, 130, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013006, , BEST, 0, 291S, 728E, 95, 961, TY, 34, NEQ, 65, 65, 65, 65, 1010, 125, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013006, , BEST, 0, 291S, 728E, 95, 961, TY, 50, NEQ, 40, 45, 45, 40, 1010, 125, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013006, , BEST, 0, 291S, 728E, 95, 961, TY, 64, NEQ, 35, 40, 35, 35, 1010, 125, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013012, , BEST, 0, 300S, 745E, 90, 965, TY, 34, NEQ, 80, 85, 70, 65, 1010, 140, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013012, , BEST, 0, 300S, 745E, 90, 965, TY, 50, NEQ, 40, 50, 45, 40, 1010, 140, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013012, , BEST, 0, 300S, 745E, 90, 965, TY, 64, NEQ, 25, 30, 25, 30, 1010, 140, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013018, , BEST, 0, 308S, 762E, 90, 965, TY, 34, NEQ, 80, 85, 70, 65, 1010, 140, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013018, , BEST, 0, 308S, 762E, 90, 965, TY, 50, NEQ, 40, 50, 45, 40, 1010, 140, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013018, , BEST, 0, 308S, 762E, 90, 965, TY, 64, NEQ, 25, 30, 25, 30, 1010, 140, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013100, , BEST, 0, 314S, 789E, 80, 972, TY, 34, NEQ, 100, 75, 110, 95, 1012, 190, 25, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013100, , BEST, 0, 314S, 789E, 80, 972, TY, 50, NEQ, 45, 30, 55, 45, 1012, 190, 25, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013100, , BEST, 0, 314S, 789E, 80, 972, TY, 64, NEQ, 25, 15, 25, 20, 1012, 190, 25, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013106, , BEST, 0, 324S, 828E, 65, 983, TY, 34, NEQ, 110, 120, 120, 110, 1010, 230, 40, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024013106, , BEST, 0, 324S, 828E, 65, 983, TY, 50, NEQ, 80, 85, 70, 65, 1010, 230, 40, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024013106, , BEST, 0, 324S, 828E, 65, 983, TY, 64, NEQ, 50, 60, 0, 0, 1010, 230, 40, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024013112, , BEST, 0, 342S, 875E, 55, 986, TS, 34, NEQ, 110, 120, 120, 110, 1006, 200, 40, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024013112, , BEST, 0, 342S, 875E, 55, 986, TS, 50, NEQ, 80, 85, 70, 65, 1006, 200, 40, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024013118, , BEST, 0, 361S, 915E, 55, 986, TS, 34, NEQ, 110, 120, 120, 110, 1006, 200, 40, 0, 0, S, 0, , 0, 0, ANGGREK, , +SH, 06, 2024013118, , BEST, 0, 361S, 915E, 55, 986, TS, 50, NEQ, 80, 85, 70, 65, 1006, 200, 40, 0, 0, S, 0, , 0, 0, ANGGREK, , diff --git a/tests/test_all.sh b/tests/test_all.sh index 0b587433d..0cf622c70 100755 --- a/tests/test_all.sh +++ b/tests/test_all.sh @@ -41,6 +41,8 @@ for call in \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ascat_noaa_25km.tc.windbarbs.imagery_windbarbs.sh" \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ascat_noaa_50km.tc.wind-ambiguities.imagery_windbarbs.sh" \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ascat_uhr.tc.wind-ambiguities.imagery_windbarbs.sh" \ + "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ascat_uhr.tc.windbarbs.imagery_windbarbs.sh" \ + "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ascat_uhr.tc.windspeed.imagery_clean.sh" \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/atms.tc.165H.netcdf_geoips.sh" \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/cli_dummy_script.sh" \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ewsg.static.Infrared.imagery_clean.sh" \ diff --git a/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.json b/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.json new file mode 100644 index 000000000..4d679fb84 --- /dev/null +++ b/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.json @@ -0,0 +1,193 @@ +{ + "name": "I_have_been_modified_more_recently_than_my_parent_yaml", + "instructions": { + "config": { + "help_str": "Various configuration-based commands for setting up your geoips environment.\nCurrently supports `geoips config install ` command.\n", + "usage_str": "To use, type `geoips config ...`\n", + "output_info": [ + "Output related to config process that was run." + ] + }, + "config_install": { + "help_str": "Install the appropriate test dataset and/or package based on the arguments\nprovided. To see a list of available test datasets for install, run\n`geoips list test-datasets`.\n", + "usage_str": "To use, type `geoips config install `.\n", + "output_info": [ + "Not Applicable" + ] + }, + "get": { + "help_str": "Retrieve an appropriate GeoIPS artifact. Currently supported `get` calls are:\n[\"family\", \"interface\", \"package\", \"plugin\"]\n", + "usage_str": "To use, type `geoips get ...`.\n", + "output_info": [ + "Information related to the artifact that was retrieved." + ] + }, + "get_family": { + "help_str": "Retrieve the appropriate GeoIPS Family alongside descriptive information of\nthat family. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS families, run:\n`geoips list interfaces`.\n", + "usage_str": "To use, type `geoips get family `, where\n is a valid GeoIPS Interface and is a supported\nfamily member of that interface.\n", + "output_info": [ + "Docstring", + "Family Name", + "Family Path", + "Interface Name", + "Interface Type", + "Required Args / Schema" + ] + }, + "get_interface": { + "help_str": "Retrieve the appropriate GeoIPS Interface alongside descriptive information of\nthat interface. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS interfaces, run:\n`geoips list interfaces`.\n", + "usage_str": "To use, type `geoips get interface `, where is a\nvalid GeoIPS Interface.\n", + "output_info": [ + "Absolute Path", + "Docstring", + "Documentation Link (If applicable)", + "Interface Name", + "Interface Type", + "Supported Families" + ] + }, + "get_package": { + "help_str": "Retrieve the appropriate GeoIPS Package alongside descriptive information of\nthat package. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS packages, run:\n`geoips list packages`.\n", + "usage_str": "To use, type `geoips get package `, where is a\nvalid GeoIPS Package.\n", + "output_info": [ + "GeoIPS Package", + "Docstring", + "Package Path", + "Documentation Link" + ] + }, + "get_plugin": { + "help_str": "Retrieve the appropriate GeoIPS Plugin alongside descriptive information of\nthat plugin. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS Plugins, run:\n`geoips list plugins`.\n", + "usage_str": "To use, type `geoips get plugin `, where\n is a valid GeoIPS Interface and is a valid plugin\nidentifier that has been implemented in any installed GeoIPS package.\n", + "output_info": [ + "Docstring", + "Documentation Link (If applicable)", + "Family", + "Interface Name", + "Package", + "Relative Path", + "Plugin Specific Info (signature / source_names / available overrides)" + ] + }, + "list": { + "help_str": "List off available GeoIPS artifacts provided further commands. Current artifacts\navailable for listing are: [\"interface\", \"interfaces\", \"packages\", \"plugins\",\n\"scripts\", \"test-datasets\", \"unit-tests\"].\n", + "usage_str": "To use, type `geoips list `.\n", + "output_info": [ + "Additional info related to the artifact[s] being listed." + ] + }, + "list_interface": { + "help_str": "List off plugins found under a certain interface. This is useful for seeing what\nplugins have been implemented through all, or a certain, GeoIPS Package. This\ninformation can then be used with\n`geoips get-plugin ` to get more information about a\ncertain plugin.\n", + "usage_str": "To use, type `geoips list interface `. Optionally includes `-p`\nflag for seeing what plugins are in a certain package (-p). For example, to see\nwhat plugins under a certain interface have been created in a single package,\nrun `geoips list interface -p `, or you can remove the\n`-p` flag to see what's been created in every package.\n", + "output_info": [ + "Family", + "Interface Name", + "Interface Type", + "Package", + "Plugin Name", + "Relative Path" + ] + }, + "list_interfaces": { + "help_str": "List off every GeoIPS Interface. This is useful for seeing what is\navailable throughout GeoIPS, but can also be extended to see what interfaces have\nbeen implemented in other geoips package, or just a certain package. This\ninformation can then be used with `geoips get-interface ` to get\nmore information about a certain interface.\n", + "usage_str": "To use, type `geoips list interfaces`. Optionally includes `-i` and `-p` flags for\nseeing what has been implemented (-i) in a certain package (-p), or all packages.\nFor example, to see what interfaces have been implemented in a certain package,\nrun `geoips list interfaces -i -p `, or you can remove the `-p` flag to\nsee what's been implemented in every package.\n", + "output_info": [ + "Absolute Path", + "Docstring", + "Documentation Link (if applicable)", + "Interface Name", + "Interface Type", + "Package", + "Supported Families" + ] + }, + "list_plugins": { + "help_str": "List off plugins found under all, or a specified GeoIPS package. This information\ncan then be used with `geoips get-plugin ` to get\nmore information about a certain plugin.\n", + "usage_str": "To use, type `geoips list plugins`. Optionally includes `-p` flag for\nseeing what plugins are in a certain package (-p). For example, to see what\nplugins have been created in a single package, run\n`geoips list plugins -p `, or you can remove the `-p` flag to see what's\nbeen created in every package.\n", + "output_info": [ + "Family", + "Interface Name", + "Interface Type", + "Package", + "Plugin Name", + "Relative Path" + ] + }, + "list_packages": { + "help_str": "List off packages found under the GeoIPS Namespace, alongside their path and\ntop-level docstring. This command will list every GeoIPS Package found, provided\nit has already been installed.\n", + "usage_str": "To use, type `geoips list packages`.\n", + "output_info": [ + "Docstring", + "Package", + "Relative Path" + ] + }, + "list_scripts": { + "help_str": "List off scripts found under all, or a specified GeoIPS package. This information\ncan then be used with `geoips run ` to run a certain\nprocess workflow, defined in a bash script.\n", + "usage_str": "To use, type `geoips list scripts`. Optionally includes `-p` flag for\nseeing what scripts are in a certain package (-p). For example, to see what\nscripts have been created in a single package, run\n`geoips list scripts -p `, or you can remove the `-p` flag to see what's\nbeen created in every package.\n", + "output_info": [ + "Package", + "Script Name" + ] + }, + "list_test-datasets": { + "help_str": "List off test-datasets used for testing GeoIPS and other GeoIPS Packages.\nThis data is used for running tests via `geoips run `.\n", + "usage_str": "To use, type `geoips list test-datasets`.\n", + "output_info": [ + "Data Host", + "Dataset Name" + ] + }, + "list_unit-tests": { + "help_str": "List off unit-tests used in a certain GeoIPS Package.\nThis data can be used to run unit tests via `geoips test unit-test ...`.\n", + "usage_str": "To use, type `geoips list unit-tests -p `.\n", + "output_info": [ + "GeoIPS Package", + "Unit Test Directory", + "Unit Test Name" + ] + }, + "run": { + "help_str": "Run a geoips process workflow (procflow), given a GeoIPS package and a script\nname. This will do the exact same thing as being in the directory of a certain\nscript, and running it via `./`. To see what scripts are available,\nyou can run `geoips list-scripts <-p>`, where `-p` can be used alongside a GeoIPS\nPackage name if you want scripts from a single package.\n", + "usage_str": "To use, type `geoips run -p `.\n", + "output_info": [ + "Log Output of the Process Workflow (Procflow)" + ] + }, + "test": { + "help_str": "Run tests implemented in GeoIPS or a separate GeoIPS Package. This could be\nrunning a set of linters on a specific package, running a certain integration or\nnormal test script, or running unit tests. Current options are [\"linting\",\n\"script\", \"unit-test\"].\n", + "usage_str": "To use, type `geoips test `.\n", + "output_info": [ + "Output of the Test Being Ran" + ] + }, + "test_linting": { + "help_str": "Run all GeoIPS linters on a specific GeoIPS Package. This will test whether or not\nthe code you wrote adheres to the specified coding conventions set up by GeoIPS.\nDefaults to the 'geoips' package.\n", + "usage_str": "To use, type `geoips test linting -p `.\n", + "output_info": [ + "Output of the Linters" + ] + }, + "test_script": { + "help_str": "Runs a specific test script (integration-based or normal), found within a certain\nGeoIPS package. If this is specified as an integration test, this will only work\nusing the 'geoips' package. Use '--integration' to specify that the script\nis integration-based. Defaults to the 'geoips' package.\n", + "usage_str": "To use, type `geoips test script -p <--integration> `.\n", + "output_info": [ + "Output of the Script Being Ran" + ] + }, + "test_unit-test": { + "help_str": "Run tests unit-test[s] implemented in a certain GeoIPS package. This command\nexpects that the unit tests implemented are pytest-based. Defaults to the 'geoips'\npackage.\n", + "usage_str": "To use, type\n`geoips test unit-test -p <-n> `\n", + "output_info": [ + "Output of the Unit Test Being Ran" + ] + }, + "validate": { + "help_str": "Validate a GeoIPS Plugin found at . While this is done under the hood via\nGeoIPS, this is an easy way to test whether or not the plugin you are developing\nis valid.\n", + "usage_str": "To use, type `geoips validate `. Do it man. NOW.\n", + "output_info": [ + "A message telling you if the plugin is valid or not and why." + ] + } + } +} \ No newline at end of file diff --git a/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.yaml b/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.yaml new file mode 100644 index 000000000..7e3486a27 --- /dev/null +++ b/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.yaml @@ -0,0 +1,263 @@ +name: geoips_cmd_instructions +instructions: + config: + help_str: | + Various configuration-based commands for setting up your geoips environment. + Currently supports `geoips config install ` command. + usage_str: | + To use, type `geoips config ...` + output_info: + - Output related to config process that was run. + config_install: + help_str: | + Install the appropriate test dataset and/or package based on the arguments + provided. To see a list of available test datasets for install, run + `geoips list test-datasets`. + usage_str: | + To use, type `geoips config install `. + output_info: + - Not Applicable + get: + help_str: | + Retrieve an appropriate GeoIPS artifact. Currently supported `get` calls are: + ["family", "interface", "package", "plugin"] + usage_str: | + To use, type `geoips get ...`. + output_info: + - Information related to the artifact that was retrieved. + get_family: + help_str: | + Retrieve the appropriate GeoIPS Family alongside descriptive information of + that family. See output_info for each datum provided when this command is + called. For a listing of available GeoIPS families, run: + `geoips list interfaces`. + usage_str: | + To use, type `geoips get family `, where + is a valid GeoIPS Interface and is a supported + family member of that interface. + output_info: + - Docstring + - Family Name + - Family Path + - Interface Name + - Interface Type + - Required Args / Schema + get_interface: + help_str: | + Retrieve the appropriate GeoIPS Interface alongside descriptive information of + that interface. See output_info for each datum provided when this command is + called. For a listing of available GeoIPS interfaces, run: + `geoips list interfaces`. + usage_str: | + To use, type `geoips get interface `, where is a + valid GeoIPS Interface. + output_info: + - Absolute Path + - Docstring + - Documentation Link (If applicable) + - Interface Name + - Interface Type + - Supported Families + get_package: + help_str: | + Retrieve the appropriate GeoIPS Package alongside descriptive information of + that package. See output_info for each datum provided when this command is + called. For a listing of available GeoIPS packages, run: + `geoips list packages`. + usage_str: | + To use, type `geoips get package `, where is a + valid GeoIPS Package. + output_info: + - GeoIPS Package + - Docstring + - Package Path + - Documentation Link + get_plugin: + help_str: | + Retrieve the appropriate GeoIPS Plugin alongside descriptive information of + that plugin. See output_info for each datum provided when this command is + called. For a listing of available GeoIPS Plugins, run: + `geoips list plugins`. + usage_str: | + To use, type `geoips get plugin `, where + is a valid GeoIPS Interface and is a valid plugin + identifier that has been implemented in any installed GeoIPS package. + output_info: + - Docstring + - Documentation Link (If applicable) + - Family + - Interface Name + - Package + - Relative Path + - Plugin Specific Info (signature / source_names / available overrides) + list: + help_str: | + List off available GeoIPS artifacts provided further commands. Current artifacts + available for listing are: ["interface", "interfaces", "packages", "plugins", + "scripts", "test-datasets", "unit-tests"]. + usage_str: | + To use, type `geoips list `. + output_info: + - Additional info related to the artifact[s] being listed. + list_interface: + help_str: | + List off plugins found under a certain interface. This is useful for seeing what + plugins have been implemented through all, or a certain, GeoIPS Package. This + information can then be used with + `geoips get-plugin ` to get more information about a + certain plugin. + usage_str: | + To use, type `geoips list interface `. Optionally includes `-p` + flag for seeing what plugins are in a certain package (-p). For example, to see + what plugins under a certain interface have been created in a single package, + run `geoips list interface -p `, or you can remove the + `-p` flag to see what's been created in every package. + output_info: + - Family + - Interface Name + - Interface Type + - Package + - Plugin Name + - Relative Path + list_interfaces: + help_str: | + List off every GeoIPS Interface. This is useful for seeing what is + available throughout GeoIPS, but can also be extended to see what interfaces have + been implemented in other geoips package, or just a certain package. This + information can then be used with `geoips get-interface ` to get + more information about a certain interface. + usage_str: | + To use, type `geoips list interfaces`. Optionally includes `-i` and `-p` flags for + seeing what has been implemented (-i) in a certain package (-p), or all packages. + For example, to see what interfaces have been implemented in a certain package, + run `geoips list interfaces -i -p `, or you can remove the `-p` flag to + see what's been implemented in every package. + output_info: + - Absolute Path + - Docstring + - Documentation Link (if applicable) + - Interface Name + - Interface Type + - Package + - Supported Families + list_plugins: + help_str: | + List off plugins found under all, or a specified GeoIPS package. This information + can then be used with `geoips get-plugin ` to get + more information about a certain plugin. + usage_str: | + To use, type `geoips list plugins`. Optionally includes `-p` flag for + seeing what plugins are in a certain package (-p). For example, to see what + plugins have been created in a single package, run + `geoips list plugins -p `, or you can remove the `-p` flag to see what's + been created in every package. + output_info: + - Family + - Interface Name + - Interface Type + - Package + - Plugin Name + - Relative Path + list_packages: + help_str: | + List off packages found under the GeoIPS Namespace, alongside their path and + top-level docstring. This command will list every GeoIPS Package found, provided + it has already been installed. + usage_str: | + To use, type `geoips list packages`. + output_info: + - Docstring + - Package + - Relative Path + list_scripts: + help_str: | + List off scripts found under all, or a specified GeoIPS package. This information + can then be used with `geoips run ` to run a certain + process workflow, defined in a bash script. + usage_str: | + To use, type `geoips list scripts`. Optionally includes `-p` flag for + seeing what scripts are in a certain package (-p). For example, to see what + scripts have been created in a single package, run + `geoips list scripts -p `, or you can remove the `-p` flag to see what's + been created in every package. + output_info: + - Package + - Script Name + list_test-datasets: + help_str: | + List off test-datasets used for testing GeoIPS and other GeoIPS Packages. + This data is used for running tests via `geoips run `. + usage_str: | + To use, type `geoips list test-datasets`. + output_info: + - Data Host + - Dataset Name + list_unit-tests: + help_str: | + List off unit-tests used in a certain GeoIPS Package. + This data can be used to run unit tests via `geoips test unit-test ...`. + usage_str: | + To use, type `geoips list unit-tests -p `. + output_info: + - GeoIPS Package + - Unit Test Directory + - Unit Test Name + run: + help_str: | + Run a geoips process workflow (procflow), given a GeoIPS package and a script + name. This will do the exact same thing as being in the directory of a certain + script, and running it via `./`. To see what scripts are available, + you can run `geoips list-scripts <-p>`, where `-p` can be used alongside a GeoIPS + Package name if you want scripts from a single package. + usage_str: | + To use, type `geoips run -p `. + output_info: + - Log Output of the Process Workflow (Procflow) + test: + help_str: | + Run tests implemented in GeoIPS or a separate GeoIPS Package. This could be + running a set of linters on a specific package, running a certain integration or + normal test script, or running unit tests. Current options are ["linting", + "script", "unit-test"]. + usage_str: | + To use, type `geoips test `. + output_info: + - Output of the Test Being Ran + test_linting: + help_str: | + Run all GeoIPS linters on a specific GeoIPS Package. This will test whether or not + the code you wrote adheres to the specified coding conventions set up by GeoIPS. + Defaults to the 'geoips' package. + usage_str: | + To use, type `geoips test linting -p `. + output_info: + - Output of the Linters + test_script: + help_str: | + Runs a specific test script (integration-based or normal), found within a certain + GeoIPS package. If this is specified as an integration test, this will only work + using the 'geoips' package. Use '--integration' to specify that the script + is integration-based. Defaults to the 'geoips' package. + usage_str: | + To use, type `geoips test script -p <--integration> `. + output_info: + - Output of the Script Being Ran + test_unit-test: + help_str: | + Run tests unit-test[s] implemented in a certain GeoIPS package. This command + expects that the unit tests implemented are pytest-based. Defaults to the 'geoips' + package. + usage_str: | + To use, type + `geoips test unit-test -p <-n> ` + output_info: + - Output of the Unit Test Being Ran + validate: + help_str: | + Validate a GeoIPS Plugin found at . While this is done under the hood via + GeoIPS, this is an easy way to test whether or not the plugin you are developing + is valid. + usage_str: | + To use, type `geoips validate `. Do it man. NOW. + output_info: + - A message telling you if the plugin is valid or not and why. diff --git a/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.json b/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.json new file mode 100644 index 000000000..bd8083ab3 --- /dev/null +++ b/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.json @@ -0,0 +1,193 @@ +{ + "name": "geoips_cmd_instructions", + "instructions": { + "config": { + "help_str": "Various configuration-based commands for setting up your geoips environment.\nCurrently supports `geoips config install ` command.\n", + "usage_str": "To use, type `geoips config ...`\n", + "output_info": [ + "Output related to config process that was run." + ] + }, + "config_install": { + "help_str": "Install the appropriate test dataset and/or package based on the arguments\nprovided. To see a list of available test datasets for install, run\n`geoips list test-datasets`.\n", + "usage_str": "To use, type `geoips config install `.\n", + "output_info": [ + "Not Applicable" + ] + }, + "get": { + "help_str": "Retrieve an appropriate GeoIPS artifact. Currently supported `get` calls are:\n[\"family\", \"interface\", \"package\", \"plugin\"]\n", + "usage_str": "To use, type `geoips get ...`.\n", + "output_info": [ + "Information related to the artifact that was retrieved." + ] + }, + "get_family": { + "help_str": "Retrieve the appropriate GeoIPS Family alongside descriptive information of\nthat family. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS families, run:\n`geoips list interfaces`.\n", + "usage_str": "To use, type `geoips get family `, where\n is a valid GeoIPS Interface and is a supported\nfamily member of that interface.\n", + "output_info": [ + "Docstring", + "Family Name", + "Family Path", + "Interface Name", + "Interface Type", + "Required Args / Schema" + ] + }, + "get_interface": { + "help_str": "Retrieve the appropriate GeoIPS Interface alongside descriptive information of\nthat interface. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS interfaces, run:\n`geoips list interfaces`.\n", + "usage_str": "To use, type `geoips get interface `, where is a\nvalid GeoIPS Interface.\n", + "output_info": [ + "Absolute Path", + "Docstring", + "Documentation Link (If applicable)", + "Interface Name", + "Interface Type", + "Supported Families" + ] + }, + "get_package": { + "help_str": "Retrieve the appropriate GeoIPS Package alongside descriptive information of\nthat package. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS packages, run:\n`geoips list packages`.\n", + "usage_str": "To use, type `geoips get package `, where is a\nvalid GeoIPS Package.\n", + "output_info": [ + "GeoIPS Package", + "Docstring", + "Package Path", + "Documentation Link" + ] + }, + "get_plugin": { + "help_str": "Retrieve the appropriate GeoIPS Plugin alongside descriptive information of\nthat plugin. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS Plugins, run:\n`geoips list plugins`.\n", + "usage_str": "To use, type `geoips get plugin `, where\n is a valid GeoIPS Interface and is a valid plugin\nidentifier that has been implemented in any installed GeoIPS package.\n", + "output_info": [ + "Docstring", + "Documentation Link (If applicable)", + "Family", + "Interface Name", + "Package", + "Relative Path", + "Plugin Specific Info (signature / source_names / available overrides)" + ] + }, + "list": { + "help_str": "List off available GeoIPS artifacts provided further commands. Current artifacts\navailable for listing are: [\"interface\", \"interfaces\", \"packages\", \"plugins\",\n\"scripts\", \"test-datasets\", \"unit-tests\"].\n", + "usage_str": "To use, type `geoips list `.\n", + "output_info": [ + "Additional info related to the artifact[s] being listed." + ] + }, + "list_interface": { + "help_str": "List off plugins found under a certain interface. This is useful for seeing what\nplugins have been implemented through all, or a certain, GeoIPS Package. This\ninformation can then be used with\n`geoips get-plugin ` to get more information about a\ncertain plugin.\n", + "usage_str": "To use, type `geoips list interface `. Optionally includes `-p`\nflag for seeing what plugins are in a certain package (-p). For example, to see\nwhat plugins under a certain interface have been created in a single package,\nrun `geoips list interface -p `, or you can remove the\n`-p` flag to see what's been created in every package.\n", + "output_info": [ + "Family", + "Interface Name", + "Interface Type", + "Package", + "Plugin Name", + "Relative Path" + ] + }, + "list_interfaces": { + "help_str": "List off every GeoIPS Interface. This is useful for seeing what is\navailable throughout GeoIPS, but can also be extended to see what interfaces have\nbeen implemented in other geoips package, or just a certain package. This\ninformation can then be used with `geoips get-interface ` to get\nmore information about a certain interface.\n", + "usage_str": "To use, type `geoips list interfaces`. Optionally includes `-i` and `-p` flags for\nseeing what has been implemented (-i) in a certain package (-p), or all packages.\nFor example, to see what interfaces have been implemented in a certain package,\nrun `geoips list interfaces -i -p `, or you can remove the `-p` flag to\nsee what's been implemented in every package.\n", + "output_info": [ + "Absolute Path", + "Docstring", + "Documentation Link (if applicable)", + "Interface Name", + "Interface Type", + "Package", + "Supported Families" + ] + }, + "list_plugins": { + "help_str": "List off plugins found under all, or a specified GeoIPS package. This information\ncan then be used with `geoips get-plugin ` to get\nmore information about a certain plugin.\n", + "usage_str": "To use, type `geoips list plugins`. Optionally includes `-p` flag for\nseeing what plugins are in a certain package (-p). For example, to see what\nplugins have been created in a single package, run\n`geoips list plugins -p `, or you can remove the `-p` flag to see what's\nbeen created in every package.\n", + "output_info": [ + "Family", + "Interface Name", + "Interface Type", + "Package", + "Plugin Name", + "Relative Path" + ] + }, + "list_packages": { + "help_str": "List off packages found under the GeoIPS Namespace, alongside their path and\ntop-level docstring. This command will list every GeoIPS Package found, provided\nit has already been installed.\n", + "usage_str": "To use, type `geoips list packages`.\n", + "output_info": [ + "Docstring", + "Package", + "Relative Path" + ] + }, + "list_scripts": { + "help_str": "List off scripts found under all, or a specified GeoIPS package. This information\ncan then be used with `geoips run ` to run a certain\nprocess workflow, defined in a bash script.\n", + "usage_str": "To use, type `geoips list scripts`. Optionally includes `-p` flag for\nseeing what scripts are in a certain package (-p). For example, to see what\nscripts have been created in a single package, run\n`geoips list scripts -p `, or you can remove the `-p` flag to see what's\nbeen created in every package.\n", + "output_info": [ + "Package", + "Script Name" + ] + }, + "list_test-datasets": { + "help_str": "List off test-datasets used for testing GeoIPS and other GeoIPS Packages.\nThis data is used for running tests via `geoips run `.\n", + "usage_str": "To use, type `geoips list test-datasets`.\n", + "output_info": [ + "Data Host", + "Dataset Name" + ] + }, + "list_unit-tests": { + "help_str": "List off unit-tests used in a certain GeoIPS Package.\nThis data can be used to run unit tests via `geoips test unit-test ...`.\n", + "usage_str": "To use, type `geoips list unit-tests -p `.\n", + "output_info": [ + "GeoIPS Package", + "Unit Test Directory", + "Unit Test Name" + ] + }, + "run": { + "help_str": "Run a geoips process workflow (procflow), given a GeoIPS package and a script\nname. This will do the exact same thing as being in the directory of a certain\nscript, and running it via `./`. To see what scripts are available,\nyou can run `geoips list-scripts <-p>`, where `-p` can be used alongside a GeoIPS\nPackage name if you want scripts from a single package.\n", + "usage_str": "To use, type `geoips run -p `.\n", + "output_info": [ + "Log Output of the Process Workflow (Procflow)" + ] + }, + "test": { + "help_str": "Run tests implemented in GeoIPS or a separate GeoIPS Package. This could be\nrunning a set of linters on a specific package, running a certain integration or\nnormal test script, or running unit tests. Current options are [\"linting\",\n\"script\", \"unit-test\"].\n", + "usage_str": "To use, type `geoips test `.\n", + "output_info": [ + "Output of the Test Being Ran" + ] + }, + "test_linting": { + "help_str": "Run all GeoIPS linters on a specific GeoIPS Package. This will test whether or not\nthe code you wrote adheres to the specified coding conventions set up by GeoIPS.\nDefaults to the 'geoips' package.\n", + "usage_str": "To use, type `geoips test linting -p `.\n", + "output_info": [ + "Output of the Linters" + ] + }, + "test_script": { + "help_str": "Runs a specific test script (integration-based or normal), found within a certain\nGeoIPS package. If this is specified as an integration test, this will only work\nusing the 'geoips' package. Use '--integration' to specify that the script\nis integration-based. Defaults to the 'geoips' package.\n", + "usage_str": "To use, type `geoips test script -p <--integration> `.\n", + "output_info": [ + "Output of the Script Being Ran" + ] + }, + "test_unit-test": { + "help_str": "Run tests unit-test[s] implemented in a certain GeoIPS package. This command\nexpects that the unit tests implemented are pytest-based. Defaults to the 'geoips'\npackage.\n", + "usage_str": "To use, type\n`geoips test unit-test -p <-n> `\n", + "output_info": [ + "Output of the Unit Test Being Ran" + ] + }, + "validate": { + "help_str": "Validate a GeoIPS Plugin found at . While this is done under the hood via\nGeoIPS, this is an easy way to test whether or not the plugin you are developing\nis valid.\n", + "usage_str": "To use, type `geoips validate `. Do it man. NOW.\n", + "output_info": [ + "A message telling you if the plugin is valid or not and why." + ] + } + } +} \ No newline at end of file diff --git a/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.yaml b/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.yaml new file mode 100644 index 000000000..aa28d3b47 --- /dev/null +++ b/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.yaml @@ -0,0 +1,412 @@ +name: 2024-05-20_19:43:53_UTC +instructions: + config: + help_str: 'Various configuration-based commands for setting up your geoips environment. + + Currently supports `geoips config install ` command. + + ' + output_info: + - Output related to config process that was run. + usage_str: 'To use, type `geoips config ...` + + ' + config_install: + help_str: 'Install the appropriate test dataset and/or package based on the arguments + + provided. To see a list of available test datasets for install, run + + `geoips list test-datasets`. + + ' + output_info: + - Not Applicable + usage_str: 'To use, type `geoips config install `. + + ' + get: + help_str: 'Retrieve an appropriate GeoIPS artifact. Currently supported `get` + calls are: + + ["family", "interface", "package", "plugin"] + + ' + output_info: + - Information related to the artifact that was retrieved. + usage_str: 'To use, type `geoips get ...`. + + ' + get_family: + help_str: 'Retrieve the appropriate GeoIPS Family alongside descriptive information + of + + that family. See output_info for each datum provided when this command is + + called. For a listing of available GeoIPS families, run: + + `geoips list interfaces`. + + ' + output_info: + - Docstring + - Family Name + - Family Path + - Interface Name + - Interface Type + - Required Args / Schema + usage_str: 'To use, type `geoips get family `, where + + is a valid GeoIPS Interface and is a supported + + family member of that interface. + + ' + get_interface: + help_str: 'Retrieve the appropriate GeoIPS Interface alongside descriptive information + of + + that interface. See output_info for each datum provided when this command is + + called. For a listing of available GeoIPS interfaces, run: + + `geoips list interfaces`. + + ' + output_info: + - Absolute Path + - Docstring + - Documentation Link (If applicable) + - Interface Name + - Interface Type + - Supported Families + usage_str: 'To use, type `geoips get interface `, where + is a + + valid GeoIPS Interface. + + ' + get_package: + help_str: 'Retrieve the appropriate GeoIPS Package alongside descriptive information + of + + that package. See output_info for each datum provided when this command is + + called. For a listing of available GeoIPS packages, run: + + `geoips list packages`. + + ' + output_info: + - GeoIPS Package + - Docstring + - Package Path + - Documentation Link + usage_str: 'To use, type `geoips get package `, where + is a + + valid GeoIPS Package. + + ' + get_plugin: + help_str: 'Retrieve the appropriate GeoIPS Plugin alongside descriptive information + of + + that plugin. See output_info for each datum provided when this command is + + called. For a listing of available GeoIPS Plugins, run: + + `geoips list plugins`. + + ' + output_info: + - Docstring + - Documentation Link (If applicable) + - Family + - Interface Name + - Package + - Relative Path + - Plugin Specific Info (signature / source_names / available overrides) + usage_str: 'To use, type `geoips get plugin `, where + + is a valid GeoIPS Interface and is a valid plugin + + identifier that has been implemented in any installed GeoIPS package. + + ' + list: + help_str: 'List off available GeoIPS artifacts provided further commands. Current + artifacts + + available for listing are: ["interface", "interfaces", "packages", "plugins", + + "scripts", "test-datasets", "unit-tests"]. + + ' + output_info: + - Additional info related to the artifact[s] being listed. + usage_str: 'To use, type `geoips list `. + + ' + list_interface: + help_str: 'List off plugins found under a certain interface. This is useful for + seeing what + + plugins have been implemented through all, or a certain, GeoIPS Package. This + + information can then be used with + + `geoips get-plugin ` to get more information about + a + + certain plugin. + + ' + output_info: + - Family + - Interface Name + - Interface Type + - Package + - Plugin Name + - Relative Path + usage_str: 'To use, type `geoips list interface `. Optionally + includes `-p` + + flag for seeing what plugins are in a certain package (-p). For example, to + see + + what plugins under a certain interface have been created in a single package, + + run `geoips list interface -p `, or you can remove + the + + `-p` flag to see what''s been created in every package. + + ' + list_interfaces: + help_str: 'List off every GeoIPS Interface. This is useful for seeing what is + + available throughout GeoIPS, but can also be extended to see what interfaces + have + + been implemented in other geoips package, or just a certain package. This + + information can then be used with `geoips get-interface ` to + get + + more information about a certain interface. + + ' + output_info: + - Absolute Path + - Docstring + - Documentation Link (if applicable) + - Interface Name + - Interface Type + - Package + - Supported Families + usage_str: 'To use, type `geoips list interfaces`. Optionally includes `-i` and + `-p` flags for + + seeing what has been implemented (-i) in a certain package (-p), or all packages. + + For example, to see what interfaces have been implemented in a certain package, + + run `geoips list interfaces -i -p `, or you can remove the `-p` flag + to + + see what''s been implemented in every package. + + ' + list_packages: + help_str: 'List off packages found under the GeoIPS Namespace, alongside their + path and + + top-level docstring. This command will list every GeoIPS Package found, provided + + it has already been installed. + + ' + output_info: + - Docstring + - Package + - Relative Path + usage_str: 'To use, type `geoips list packages`. + + ' + list_plugins: + help_str: 'List off plugins found under all, or a specified GeoIPS package. This + information + + can then be used with `geoips get-plugin ` to + get + + more information about a certain plugin. + + ' + output_info: + - Family + - Interface Name + - Interface Type + - Package + - Plugin Name + - Relative Path + usage_str: 'To use, type `geoips list plugins`. Optionally includes `-p` flag + for + + seeing what plugins are in a certain package (-p). For example, to see what + + plugins have been created in a single package, run + + `geoips list plugins -p `, or you can remove the `-p` flag to see + what''s + + been created in every package. + + ' + list_scripts: + help_str: 'List off scripts found under all, or a specified GeoIPS package. This + information + + can then be used with `geoips run ` to run a certain + + process workflow, defined in a bash script. + + ' + output_info: + - Package + - Script Name + usage_str: 'To use, type `geoips list scripts`. Optionally includes `-p` flag + for + + seeing what scripts are in a certain package (-p). For example, to see what + + scripts have been created in a single package, run + + `geoips list scripts -p `, or you can remove the `-p` flag to see + what''s + + been created in every package. + + ' + list_test-datasets: + help_str: 'List off test-datasets used for testing GeoIPS and other GeoIPS Packages. + + This data is used for running tests via `geoips run `. + + ' + output_info: + - Data Host + - Dataset Name + usage_str: 'To use, type `geoips list test-datasets`. + + ' + list_unit-tests: + help_str: 'List off unit-tests used in a certain GeoIPS Package. + + This data can be used to run unit tests via `geoips test unit-test ...`. + + ' + output_info: + - GeoIPS Package + - Unit Test Directory + - Unit Test Name + usage_str: 'To use, type `geoips list unit-tests -p `. + + ' + run: + help_str: 'Run a geoips process workflow (procflow), given a GeoIPS package and + a script + + name. This will do the exact same thing as being in the directory of a certain + + script, and running it via `./`. To see what scripts are available, + + you can run `geoips list-scripts <-p>`, where `-p` can be used alongside a GeoIPS + + Package name if you want scripts from a single package. + + ' + output_info: + - Log Output of the Process Workflow (Procflow) + usage_str: 'To use, type `geoips run -p `. + + ' + test: + help_str: 'Run tests implemented in GeoIPS or a separate GeoIPS Package. This + could be + + running a set of linters on a specific package, running a certain integration + or + + normal test script, or running unit tests. Current options are ["linting", + + "script", "unit-test"]. + + ' + output_info: + - Output of the Test Being Ran + usage_str: 'To use, type `geoips test `. + + ' + test_linting: + help_str: 'Run all GeoIPS linters on a specific GeoIPS Package. This will test + whether or not + + the code you wrote adheres to the specified coding conventions set up by GeoIPS. + + Defaults to the ''geoips'' package. + + ' + output_info: + - Output of the Linters + usage_str: 'To use, type `geoips test linting -p `. + + ' + test_script: + help_str: 'Runs a specific test script (integration-based or normal), found within + a certain + + GeoIPS package. If this is specified as an integration test, this will only + work + + using the ''geoips'' package. Use ''--integration'' to specify that the script + + is integration-based. Defaults to the ''geoips'' package. + + ' + output_info: + - Output of the Script Being Ran + usage_str: 'To use, type `geoips test script -p <--integration> + `. + + ' + test_unit-test: + help_str: 'Run tests unit-test[s] implemented in a certain GeoIPS package. This + command + + expects that the unit tests implemented are pytest-based. Defaults to the ''geoips'' + + package. + + ' + output_info: + - Output of the Unit Test Being Ran + usage_str: 'To use, type + + `geoips test unit-test -p <-n> ` + + ' + validate: + help_str: 'Validate a GeoIPS Plugin found at . While this is done under + the hood via + + GeoIPS, this is an easy way to test whether or not the plugin you are developing + + is valid. + + ' + output_info: + - A message telling you if the plugin is valid or not and why. + usage_str: 'To use, type `geoips validate `. Do it man. NOW. + + ' diff --git a/tests/unit_tests/commandline/test_expose.py b/tests/unit_tests/commandline/test_expose.py index 52b16f3e6..8695eb3d5 100644 --- a/tests/unit_tests/commandline/test_expose.py +++ b/tests/unit_tests/commandline/test_expose.py @@ -1,3 +1,6 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + """Unit test asserting functionality for exposing plugin-package commands.""" from importlib import metadata diff --git a/tests/unit_tests/commandline/test_geoips_test_sector.py b/tests/unit_tests/commandline/test_geoips_test_sector.py index 66f3d8f50..6625e10c4 100644 --- a/tests/unit_tests/commandline/test_geoips_test_sector.py +++ b/tests/unit_tests/commandline/test_geoips_test_sector.py @@ -1,14 +1,5 @@ -# # # Distribution Statement A. Approved for public release. Distribution is unlimited. -# # # -# # # Author: -# # # Naval Research Laboratory, Marine Meteorology Division -# # # -# # # This program is free software: you can redistribute it and/or modify it under -# # # the terms of the NRLMMD License included with this program. This program is -# # # distributed WITHOUT ANY WARRANTY; without even the implied warranty of -# # # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the included license -# # # for more details. If you did not receive the license, for more information see: -# # # https://github.com/U-S-NRL-Marine-Meteorology-Division/ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. """Unit test for GeoIPS CLI `test sector` command. diff --git a/tests/unit_tests/commandline/test_geoips_tree.py b/tests/unit_tests/commandline/test_geoips_tree.py index a76be8ded..6a2acde7c 100644 --- a/tests/unit_tests/commandline/test_geoips_tree.py +++ b/tests/unit_tests/commandline/test_geoips_tree.py @@ -1,3 +1,6 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + """Unit test for GeoIPS CLI `tree` command. See geoips/commandline/ancillary_info/cmd_instructions.yaml for more information. diff --git a/tests/utils/check_code.sh b/tests/utils/check_code.sh index 0d738a32b..9fb5b0798 100755 --- a/tests/utils/check_code.sh +++ b/tests/utils/check_code.sh @@ -176,6 +176,7 @@ if [[ "$test" == "pytest_short" || "$test" == "all" ]]; then fi echo "TEST COMPLETE pytest_short" fi +# Do not include in "test" keyword - this takes a very long time to run if [[ "$test" == "pytest_long" || "$test" == "all_test_data" ]]; then echo "" echo "CALLING TEST:"