diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..90b48ec73 --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +max-line-length=88 +count=True +ignore=E203,W503,E712 +extend-exclude=_version.py,lib,*_docs,geoips_dev_utils +docstring-convention=numpy +rst-roles=class,func,ref +rst-directives=envvar,exception +rst-substitutions=version +statistics=True +per-file-ignores = + /*/interfaces/__init__.py:F401 diff --git a/.github/versions/tagged_version b/.github/versions/tagged_version new file mode 100644 index 000000000..63e799cf4 --- /dev/null +++ b/.github/versions/tagged_version @@ -0,0 +1 @@ +1.14.1 diff --git a/.github/versions/upcoming_version b/.github/versions/upcoming_version new file mode 100644 index 000000000..4ea8ad87e --- /dev/null +++ b/.github/versions/upcoming_version @@ -0,0 +1 @@ +1.14.3 diff --git a/.github/workflows/brassy-notes.yaml b/.github/workflows/brassy-notes.yaml new file mode 100644 index 000000000..d91e01c33 --- /dev/null +++ b/.github/workflows/brassy-notes.yaml @@ -0,0 +1,25 @@ +name: Brassy + +on: + # Triggers the workflow when pull request closed (either closed + # as non-planned, or actually approved and merged.) + pull_request: + types: + - closed + branch: main + # Allows run of this workflow manually from the Actions tab + # Must be merged to default before it will be available to manually run. + workflow_dispatch: + +jobs: + brassy-notes: + name: Brassy + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-brassy-notes.yaml@main + # Only run this if the pull request was merged, not just closed. + if: github.event.pull_request.merged == true + permissions: + contents: write + pull-requests: write + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy-docs.yaml b/.github/workflows/deploy-docs.yaml new file mode 100644 index 000000000..b32188262 --- /dev/null +++ b/.github/workflows/deploy-docs.yaml @@ -0,0 +1,36 @@ +name: Deploy docs + +# 1. Brassy - on merge of v*-version-release +# * generate release notes +# * commit and push to v*-add-rst-release-note +# * open PR to default branch +# 2. Tag and Release - on merge of v*-add-rst-release-note +# * Tag current version +# * Release just tagged version +# 3. Package and Publish - on published release (from #2) +# * Build wheel +# * Publish to pypi +# 4. Deploy docs - on published release (from #2) +# * pip install geoips +# * pip install plugin repo +# * build docs with geoips/docs/build_docs.sh +# * deploy docs with geoips/docs/deploy_pages.sh + +on: + # Triggers the workflow on published release + release: + types: + - published + # Allows run of this workflow manually from the Actions tab + # Must be merged to default before it will be available to manually run. + workflow_dispatch: + +jobs: + deploy-docs: + name: Deploy docs + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-deploy-docs.yaml@main + permissions: + contents: write + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy-ghpages-docs.yaml b/.github/workflows/deploy-ghpages-docs.yaml deleted file mode 100644 index 266ab8fde..000000000 --- a/.github/workflows/deploy-ghpages-docs.yaml +++ /dev/null @@ -1,133 +0,0 @@ -name: Build and deploy docs - -defaults: - run: - shell: bash - -on: - # Allows run of this workflow manually from the Actions tab - # Must be merged to dev before it will be available to manually run. - workflow_dispatch: - # Run on each published release - release: - types: [published] - -jobs: - deploy_ghpages: - runs-on: ${{ vars.RUNNER }} - steps: - - name: Setup Python - uses: actions/setup-python@v4 # v4is the latest for setup-python - with: - python-version: "pypy-3.10" # I believe 3.10 is the latest for actions - check-latest: true - - name: Update pip - run: pip install --upgrade pip - - name: Checkout geoips repo default branch - uses: actions/checkout@v3 # v3 is the latest for checkout - with: - fetch-depth: 0 - repository: ${{ vars.ORGANIZATION }}/geoips - ref: ${{ vars.DEFAULT_BRANCH }} - token: ${{ secrets.GEOIPS_TOKEN }} - path: geoips_default_branch - - name: pip install geoips_default_branch[doc] - run: | - which pip - pip uninstall -y geoips - pip install $PWD/geoips_default_branch[doc] - ret=$? - echo "${ret}" - python -c "import geoips; print(geoips.__version__)" - - name: Checkout current repo current branch - uses: actions/checkout@v3 # v3 is the latest for checkout - with: - fetch-depth: 0 - path: geoips_plugin_repo - - name: pip install geoips plugin repo - run: | - which pip - curr_repo=`echo "$GITHUB_REPOSITORY" | sed "s,$GITHUB_REPOSITORY_OWNER/,,"` - # Do not re-install if this is the geoips repo - if [[ "$curr_repo" != "geoips" ]]; then - pip uninstall -y $curr_repo - pip install -v $PWD/geoips_plugin_repo - ret=$? - echo "${ret}" - python -c "import $curr_repo; print($curr_repo.__version__)" - fi - - name: create_plugin_registries - run: | - echo "create plugin registries" - which create_plugin_registries - create_plugin_registries - ret=$? - echo "${ret}" - - name: Run build docs - run: | - which pip - echo "Build html and pdf docs" - # Ie, get 'recenter_tc' out of 'NRLMMD-GEOIPS/recenter_tc' - curr_repo=`echo "$GITHUB_REPOSITORY" | sed "s,$GITHUB_REPOSITORY_OWNER/,,"` - # Call build_docs.sh from the geoips default branch - # Pass in - # the path to the current plugin repo, - # the name of the current repo, - # "html_pdf" to indicate producing both html and pdf output, and - # the path to the docs template directory in the geoips default branch. - ./geoips_default_branch/docs/build_docs.sh ./geoips_plugin_repo $curr_repo html_pdf ./geoips_default_branch/docs - ret=$? - if [[ "${ret##*:}" != *"0"* ]]; then - echo "::error::Building html and pdf docs ${ret##*:}" - exit 1 - fi - - name: Run deploy docs - run: | - which pip - echo "Deploy docs to github pages" - # GITHUB_REPOSITORY, GITHUB_REPOSITORY_OWNER, and GITHUB_SERVER_URL are all - # standard github variables defined for every github actions run. - # Ie, get 'recenter_tc' out of 'NRLMMD-GEOIPS/recenter_tc' - curr_repo=`echo "$GITHUB_REPOSITORY" | sed "s,$GITHUB_REPOSITORY_OWNER/,,"` - # ie, https://github.com/NRLMMD-GEOIPS - GEOIPS_REPO_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY_OWNER" - # Remove ghpages_repo dir if it exists. - # deploy_pages.sh will not run if ghpages_repo exists. - if [[ -d ./ghpages_repo ]]; then - echo "::group::remove_ghpages_before" - echo "Remove ghpages_repo before deploy" - echo "rm -rfv ./ghpages_repo" - rm -rfv ./ghpages_repo - echo "::endgroup::" - rmret=$? - if [[ "${rmret##*:}" != *"0"* ]]; then - echo "::error::Removing ./ghpages_repo before deploy ${rmret##*:}" - exit 1 - fi - fi - # Use the deploy_pages.sh script from the geoips default branch. - # Pass in the - # current plugin repo path, - # the geoips repo url, and - # a path to a new directory for the ghpages checkout/push - # the name of the current repository - ./geoips_default_branch/docs/deploy_pages.sh ./geoips_plugin_repo $GEOIPS_REPO_URL ./ghpages_repo $curr_repo - ret=$? - if [[ "${ret##*:}" != *"0"* ]]; then - echo "::error::Deploying docs to GitHub pages ${ret##*:}" - exit 1 - fi - # Remove ghpages_repo dir after running deploy_pages.sh. - # deploy_pages.sh will not run if ghpages_repo exists. - if [[ -d ./ghpages_repo ]]; then - echo "::group::remove_ghpages_after" - echo "Remove ghpages_repo after deploy" - echo "rm -rfv ./ghpages_repo" - rm -rfv ./ghpages_repo - echo "::endgroup::" - rmret=$? - if [[ "${rmret##*:}" != *"0"* ]]; then - echo "::error::Removing ./ghpages_repo after deploy ${rmret##*:}" - exit 1 - fi - fi diff --git a/.github/workflows/doc-lint-test.yaml b/.github/workflows/doc-lint-test.yaml deleted file mode 100644 index e771245f3..000000000 --- a/.github/workflows/doc-lint-test.yaml +++ /dev/null @@ -1,644 +0,0 @@ -# Builds an image that contains GeoIPS built with `pip install .[doc,lint,test]` -name: Build Documentation, Lint, and Test - -env: - # The registry that will be used to store all images - # Override from repo or org level to use a different registry - DOCKER_REGISTRY: ghcr.io - # The path to the GeoIPS package in DOCLINTTEST images - BASE_GEOIPS_PATH: /packages/geoips - # Allow overriding the doclinttest image tag used for plugin packages - # Allows using a different version of the doc, lint, and test code - PLUGIN_DOCLINTTEST_TAG: doclinttest-stable - -defaults: - run: - shell: bash - -on: - # Triggers when code is pushed to "main" branch - # This should only happen when PRs are merged - push: - branches: - - main - - # Triggers the workflow when pull request created and updated - pull_request: - - # Triggers when a new release is created - release: - types: - - published - - # Allows run of this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - # Set variables that should be accessible to all jobs - set_variables: - runs-on: ${{ vars.RUNNER }} - env: - CURR_REPO: ${{ github.event.repository.name }} - outputs: - # The name of the untagged docker images to be created by this workflow - IMAGE_NAME: ${{ steps.set_image_name.outputs.IMAGE_NAME }} - # The tag to use for the doclinttest images - DOCLINTTEST_TAG: ${{ steps.set_doclinttest_tag.outputs.DOCLINTTEST_TAG }} - # The image to use for doclinttest jobs - DOCLINTTEST_IMAGE: ${{ steps.set_doclinttest_image.outputs.DOCLINTTEST_IMAGE }} - # Flags for enabling/disabling various parts of the workflow - # Default to enabled unless overridden by a repository-level variable - CI_BUILD_SPHINX_HTML: ${{ steps.get_job_flags.outputs.CI_BUILD_SPHINX_HTML }} - CHECK_NEW_RELEASE_NOTE: ${{ steps.get_job_flags.outputs.CHECK_NEW_RELEASE_NOTE }} - CI_BUILD_SPHINX_PDF: ${{ steps.get_job_flags.outputs.CI_BUILD_SPHINX_PDF }} - CI_LINT_BANDIT: ${{ steps.get_job_flags.outputs.CI_LINT_BANDIT }} - CI_LINT_DOC8: ${{ steps.get_job_flags.outputs.CI_LINT_DOC8 }} - CI_LINT_FLAKE8: ${{ steps.get_job_flags.outputs.CI_LINT_FLAKE8 }} - CI_LINT_BLACK: ${{ steps.get_job_flags.outputs.CI_LINT_BLACK }} - CI_TEST_INTERFACES: ${{ steps.get_job_flags.outputs.CI_TEST_INTERFACES }} - CI_TEST_PYTEST_SHORT: ${{ steps.get_job_flags.outputs.CI_TEST_PYTEST_SHORT }} - steps: - - name: Set the image name - id: set_image_name - run: | - # Create an image name using the organiztion and repository names, lowercase - image_name=${DOCKER_REGISTRY}/${GITHUB_REPOSITORY,,} - echo "IMAGE_NAME=${image_name}" - echo "IMAGE_NAME=${image_name}" >> ${GITHUB_ENV} - echo "IMAGE_NAME=${image_name}" >> ${GITHUB_OUTPUT} - - name: Set doclinttest image tag - id: set_doclinttest_tag - run: | - echo "CURR_REPO: ${CURR_REPO}" - # If the current repository is geoips, use the image we're about to build - # for running tests and building docs. - if [[ "${CURR_REPO}" == "geoips" ]]; then - echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" - echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" >> ${GITHUB_ENV} - echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" >> ${GITHUB_OUTPUT} - else - echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" - echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" >> ${GITHUB_ENV} - echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" >> ${GITHUB_OUTPUT} - fi - - name: Set doclinttest image - id: set_doclinttest_image - run: | - owner=${{ github.repository_owner }} - owner=${owner,,} - doclinttest_image_name=${DOCKER_REGISTRY}/${owner}/geoips:${DOCLINTTEST_TAG} - echo "DOCLINTTEST_IMAGE=${DOCKER_REGISTRY}/${owner}/geoips:${DOCLINTTEST_TAG}" - echo "DOCLINTTEST_IMAGE=${DOCKER_REGISTRY}/${owner}/geoips:${DOCLINTTEST_TAG}" >> ${GITHUB_ENV} - echo "DOCLINTTEST_IMAGE=${DOCKER_REGISTRY}/${owner}/geoips:${DOCLINTTEST_TAG}" >> ${GITHUB_OUTPUT} - - name: Get job flags - id: get_job_flags - run: | - # Define an array of flags and their corresponding secret values - flags=(CHECK_NEW_RELEASE_NOTE\ - CI_BUILD_SPHINX_HTML \ - CI_BUILD_SPHINX_PDF \ - CI_LINT_BANDIT \ - CI_LINT_DOC8 \ - CI_LINT_FLAKE8 \ - CI_LINT_BLACK \ - CI_TEST_INTERFACES \ - CI_TEST_PYTEST_SHORT) - for flag in "${flags[@]}"; do - # Dynamically construct the variable name and access the corresponding secret - case $flag in - CHECK_NEW_RELEASE_NOTE) val="${{ vars.CHECK_NEW_RELEASE_NOTE }}" ;; - CI_BUILD_SPHINX_HTML) val="${{ vars.CI_BUILD_SPHINX_HTML }}" ;; - CI_BUILD_SPHINX_PDF) val="${{ vars.CI_BUILD_SPHINX_PDF }}" ;; - CI_LINT_BANDIT) val="${{ vars.CI_LINT_BANDIT }}" ;; - CI_LINT_DOC8) val="${{ vars.CI_LINT_DOC8 }}" ;; - CI_LINT_FLAKE8) val="${{ vars.CI_LINT_FLAKE8 }}" ;; - CI_LINT_BLACK) val="${{ vars.CI_LINT_BLACK }}" ;; - CI_TEST_INTERFACES) val="${{ vars.CI_TEST_INTERFACES }}" ;; - CI_TEST_PYTEST_SHORT) val="${{ vars.CI_TEST_PYTEST_SHORT }}" ;; - esac - # Check if the secret value is "true" and set the environment variable and output - if [ "$val" == "false" ]; then - echo "${flag}=false" - echo "${flag}=false" >> $GITHUB_ENV - echo "${flag}=false" >> $GITHUB_OUTPUT - else - echo "${flag}=true" - echo "${flag}=true" >> $GITHUB_ENV - echo "${flag}=true" >> $GITHUB_OUTPUT - fi - done - - print_run_flags: - runs-on: ${{ vars.RUNNER }} - needs: [set_variables] - steps: - - name: Print run flags - run: | - for var in $(echo '${{ toJson(needs.set_variables.outputs) }}' | jq -r 'keys[]'); do - value=$(echo '${{ toJson(needs.set_variables.outputs) }}' | jq -r --arg var "$var" '.[$var]') - echo "$var=$value" - done - - # Build the base GeoIPS doclinttest image - # This image is used in all subsequent tests - # It is only built for the main GeoIPS package but is used by all repositories - # - # Within this image, the GeoIPS package is located in /packages/geoips and is - # installed using `pip install .[doc,lint,test]`. Scripts from geoips can be called - # from /packages/geoips. - build_doclinttest_image: - runs-on: ${{ vars.RUNNER }} - needs: set_variables - if: ${{ github.event.repository.name == 'geoips' }} - env: - IMAGE_NAME: ${{ needs.set_variables.outputs.IMAGE_NAME }} - BUILDCACHE_IMAGE: ${{ needs.set_variables.outputs.IMAGE_NAME }}:buildcache - DOCLINTTEST_IMAGE: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - # Can't write to registry without this - permissions: - contents: read - packages: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup BuildX - uses: docker/setup-buildx-action@v3 - - - name: Login to the Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.DOCKER_REGISTRY }} - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - - # This cache allows the second push below when on the main branch - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Create tags - run: | - tags="${DOCLINTTEST_IMAGE},${BUILDCACHE_IMAGE}" - if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then - tags="${tags},${IMAGE_NAME}:doclinttest-stable" - else - tags="${tags},${IMAGE_NAME}:doclinttest-latest" - fi - echo "PUSH_TAGS=${tags}" - echo "PUSH_TAGS=${tags}" >> $GITHUB_ENV - - - name: Build and Push image - uses: docker/build-push-action@v6 - with: - context: . - target: doclinttest - push: true - tags: ${{ env.PUSH_TAGS }} - file: "Dockerfile" - cache-from: type=registry,ref=${{ env.BUILDCACHE_IMAGE }} - cache-to: type=registry,ref=ghcr.io/${{ env.BUILDCACHE_IMAGE }},mode=max - - -################################################# -# OLD Release Note NOT Added -################################################# - check_no_edits_to_rst_release_note: - if: > - always() && - needs.set_variables.outputs.CHECK_NEW_RELEASE_NOTE == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - env: - CURR_REPO: ${{ github.event.repository.name }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Configure Git safe directory - run: git config --global --add safe.directory $PWD - - name: Detect new and modified release files - run: | - #update_this_file="$(cat update_this_release_note)" - release_notes_dir="$PWD/docs/source/releases" #/$(dirname $update_this_file)" - echo "Checking for no edits to the .rst release note in dir $release_notes_dir" - echo "git -C . diff --name-only --diff-filter=AM remotes/origin/main -- $release_notes_dir/*.rst" - ret=$(git -C . diff --name-only --diff-filter=AM remotes/origin/main -- $release_notes_dir/*.rst) - for file in ${ret[@]}; do - echo " $file" - done - if [ -z "$ret" ]; then - echo "PASSED" - else - echo "FAILED: Release note changes detected" - echo "Please use brassy to enter change logs in $release_notes_dir" - echo "DO NOT EDIT .RST FILES DIRECTLY :))" - exit 1 - fi - -################################################# -# Release Note Added -################################################# - check_new_release_note: - if: > - always() && - github.ref != 'refs/heads/main' && - needs.set_variables.outputs.CHECK_NEW_RELEASE_NOTE == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - env: - CURR_REPO: ${{ github.event.repository.name }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Configure Git safe directory - run: git config --global --add safe.directory $PWD - - name: Detect new and modified release files - run: | - echo "Checking for new and modified release note files in $PWD" - #update_this_file="$(cat update_this_release_note)" - release_notes_dir="$PWD/docs/source/releases/latest" #/$(dirname $update_this_file)" - ls $release_notes_dir - #current_release_note=`cat update_this_release_note` - echo "git -C . diff --name-only --diff-filter=AM remotes/origin/main -- $release_notes_dir/*.yaml" - ret=$(git -C . diff --name-only --diff-filter=AM remotes/origin/main -- $release_notes_dir/*.yaml) - echo "Release note files modified this PR:" - for file in ${ret[@]}; do - echo " $file" - done - if [ -z "$ret" ]; then - echo "FAILED: No yaml release note changes detected" - echo "Please use brassy to enter change logs in $release_notes_dir" - exit 1 - else - echo "PASSED" - fi - -################################################# -# Documentation builds -################################################# - build_sphinx_html: - if: > - always() && - needs.set_variables.outputs.CI_BUILD_SPHINX_HTML == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - env: - CURR_REPO: ${{ github.event.repository.name }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Configure Git safe directory - run: git config --global --add safe.directory $PWD - - - name: Install package - run: pip install . - - - name: Build release notes from yaml files - id: release-note-generator - run: | - # update_this_file="$(cat $BASE_GEOIPS_PATH/update_this_release_note)" - # release_notes_dir="$(dirname $update_this_file)/$(basename $update_this_file .rst)" - release_notes_dir="$PWD/docs/source/releases/latest" #/$(dirname $update_this_file)" - update_this_file="$PWD/docs/source/releases/latest.rst" - #release_version="$(basename $update_this_file .rst)" - release_version="latest" - echo "Running brassy on directory: $release_notes_dir" - brassy --release-version $release_version --no-rich --output-file $update_this_file $release_notes_dir - echo "Done writing!" - echo "release_note_file=$update_this_file" >> $GITHUB_OUTPUT - - name: Pinken release note - run: pink ${{ steps.release-note-generator.outputs.release_note_file }} - - - name: Upload release note - uses: actions/upload-artifact@v4 - with: - name: Release Note - path: ${{ steps.release-note-generator.outputs.release_note_file }} - if-no-files-found: error - - - name: Run build html - id: build-html-docs - run: | - echo "Build html docs" - # Call build_docs.sh from the geoips default branch - # Pass in - # the path to the current plugin repo, - # the name of the current repo, - # "html_only" to indicate producing only html output, and - # the path to the docs template directory in the geoips default branch. - $BASE_GEOIPS_PATH/docs/build_docs.sh . $CURR_REPO html_only $BASE_GEOIPS_PATH/docs - ret=$? - if [[ "${ret##*:}" != *"0"* ]]; then - echo "::error::Building html docs ${ret##*:}" - exit 1 - fi - echo "BUILT_DOCS_PATH=$PWD/build/sphinx/html" >> $GITHUB_OUTPUT - - name: Upload HTML Docs - uses: actions/upload-artifact@v4 - with: - name: HTML Docs - path: ${{ steps.build-html-docs.outputs.BUILT_DOCS_PATH }} - if-no-files-found: error - -# This job does not currently work because latex is not installed in the image. I think -# that to get this job working would only require installing texlive-base but other -# texlive packages might also be required. -# -# Additionally, this job is extremely slow. Building the PDF documentation is time -# consuming and should probably only be run on releases. Building the HTML documentation -# should be enough to catch any problems that crop up. -# -# build_sphinx_pdf: -# if: ${{ needs.set_variables.outputs.CI_DISABLE_BUILD_SPHINX_PDF == 'false' }} -# runs-on: ${{ vars.RUNNER }} -# needs: [build_doclinttest_image, set_variables] -# container: -# image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} -# credentials: -# username: ${{ secrets.DOCKER_REGISTRY_USER }} -# password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} -# env: -# CURR_REPO: ${{ github.event.repository.name }} -# steps: -# - name: Checkout code -# uses: actions/checkout@v4 -# -# - name: Configure Git safe directory -# run: git config --global --add safe.directory $PWD -# -# - name: Install geoips documentation packages -# run: pip install -e .[doc] -# -# - name: Run build pdf -# run: | -# echo "Build pdf docs" -# # Call build_docs.sh from the geoips default branch -# # Pass in -# # the path to the current plugin repo, -# # the name of the current repo, -# # "pdf_only" to indicate producing only pdf output, and -# # the path to the docs template directory in the geoips default branch. -# ./docs/build_docs.sh . $CURR_REPO pdf_only ./docs -# ret=$? -# if [[ "${ret##*:}" != *"0"* ]]; then -# echo "::error::Building pdf docs ${ret##*:}" -# exit 1 -# fi - -################################################# -# Linting -################################################# - lint_bandit: - if: > - always() && - needs.set_variables.outputs.CI_LINT_BANDIT == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Run bandit - shell: bash -l {0} - run: | - echo "::group::bandit_analysis" - echo "BANDIT analysis of code" - $BASE_GEOIPS_PATH/tests/utils/check_code.sh bandit ./ - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to bandit violations, return code ${ret}" - exit 1 - fi - - lint_doc8: - if: > - always() && - needs.set_variables.outputs.CI_LINT_DOC8 == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Run doc8 - shell: bash -l {0} - run: | - echo "::group::doc8_analysis" - if [ ! -d "./docs/source/new-docs" ]; then - echo "No new-docs directory found in ${CURR_REPO}" - exit 0 - fi - echo "Doc8 analysis of code" - doc8 --max-line-length=120 ./docs/source/new-docs/ - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to doc8 violations, return code ${ret}" - exit 1 - fi - - lint_flake8: - if: > - always() && - needs.set_variables.outputs.CI_LINT_FLAKE8 == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Run flake8 - shell: bash -l {0} - run: | - echo "::group::flake8_analysis" - echo "FLAKE8 analysis of code" - $BASE_GEOIPS_PATH/tests/utils/check_code.sh flake8 . - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to flake8 violations, return code ${ret}" - exit 1 - fi - - lint_black: - if: > - always() && - needs.set_variables.outputs.CI_LINT_BLACK == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Run code check script black - shell: bash -l {0} - run: | - echo "::group::black_analysis" - echo "BLACK analysis of code" - black --version - $BASE_GEOIPS_PATH/tests/utils/check_code.sh black ./ - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to black violations, return code ${ret}" - exit 1 - fi - -################################################# -# Code tests -################################################# - test_interfaces: - if: > - always() && - needs.set_variables.outputs.CI_TEST_INTERFACES == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Configure Git safe directory - run: git config --global --add safe.directory $PWD - - name: Pip install package - run: pip install -v . - - name: create_plugin_registries - run: create_plugin_registries - - name: Run code check script interfaces - run: | - $BASE_GEOIPS_PATH/tests/utils/check_code.sh interfaces geoips - ret=$? - echo "::group::interface_analysis" - echo "INTERFACES analysis of code" - echo "${ret}" - echo "::endgroup::" - if [[ "${ret##*:}" != *"0"* ]]; then - echo "::error::due to interface violations ${ret##*:}" - exit 1 - fi - - test_pytest_short: - if: > - always() && - needs.set_variables.outputs.CI_TEST_PYTEST_SHORT == 'true' && - (needs.build_doclinttest_image.result == 'success' || - needs.build_doclinttest_image.result == 'skipped') - runs-on: ${{ vars.RUNNER }} - needs: [build_doclinttest_image, set_variables] - container: - image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} - credentials: - username: ${{ secrets.DOCKER_REGISTRY_USER }} - password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} - env: - CURR_REPO: ${{ github.event.repository.name }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Configure Git safe directory - run: git config --global --add safe.directory $PWD - - name: Pip install package - run: pip install -v . - - name: create_plugin_registries - run: create_plugin_registries - # Run unit tests for this repo if they exist - - name: Run pytest unit tests - run: | - echo "::group::pytest_unit_test" - echo "Pytest short unit tests of ${CURR_REPO}" - echo "which pytest" - which pytest - if [ ! -d "./tests/unit_tests" ]; then - echo "No unit tests found in ${CURR_REPO}" - exit 0 - fi - echo "pytest -q ./tests/unit_tests" - pytest -q ./tests/unit_tests - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to geoips repo pytest errors, return code ${ret}" - exit 1 - fi - - name: Run base GeoIPS pytest unit tests - if: ${{ env.CURR_REPO != 'geoips' }} - run: | - echo "::group::pytest_unit_test" - echo "Pytest short unit tests of ${CURR_REPO} using core GeoIPS tests" - echo "which pytest" - which pytest - echo "pytest -q $BASE_GEOIPS_PATH/tests/unit_tests" - pytest -q $BASE_GEOIPS_PATH/tests/unit_tests - ret=$? - echo "Return code: ${ret}" - echo "::endgroup::" - if [[ "${ret}" != *"0"* ]]; then - echo "::error::due to geoips repo pytest errors, return code ${ret}" - exit 1 - fi diff --git a/.github/workflows/doc-test.yaml b/.github/workflows/doc-test.yaml new file mode 100644 index 000000000..7ada5d81e --- /dev/null +++ b/.github/workflows/doc-test.yaml @@ -0,0 +1,477 @@ +# Builds an image that contains GeoIPS built with `pip install .[doc,lint,test]` +name: Build Documentation and Test + +# NOTE: Various bits of CI run on particularly-named branches +# v*-version-release : auto-generated final release branch, +# merge kicks off new-brassy-note +# v*-update-rst-release-note: auto-generated branch in new-brassy-note + +env: + # REQUIRED organization level variables used in this workflow. + # ${{ github.event.repository.default_branch }} is used throughout + # rather than main to generalize default branch + # (removed DEFAULT_BRANCH org var) + # DOCKER_REGISTRY: URL to the docker registry + # DOCKER_REGISTRY_USER: user to sign into DOCKER_REGISTRY + # DOCKER_REGISTRY_TOKEN: token for USER user to sign into DOCKER_REGISTRY + # RUNNER: default runner + # UPLOAD_GITHUB_ARTIFACTS: true/false, specify whether to upload artifacts to github + # USE_DOCKER_BUILDX_ACTION: true/false, specify whether to use the docker buildx action + + # OPTIONAL organization level variables: + # # Some system implementations may require different runners for building and + # # running docker containers - allow specifying separately. Defaults to vars.RUNNER + # # if these are not defined. Note env context nor strings are supported in the + # # runs-on field, so we MUST use the vars context for the default. + # RUNNER_DOCKERBUILD: runner to build docker containers + # RUNNER_DOCKERRUN: runner to run docker containers + + # The path to the GeoIPS package in DOCLINTTEST images + BASE_GEOIPS_PATH: /packages/geoips + # Allow overriding the doclinttest image tag used for plugin packages + # Allows using a different version of the doc, lint, and test code + PLUGIN_DOCLINTTEST_TAG: ${{ vars.PLUGIN_DOCLINTTEST_TAG || 'doclinttest-stable' }} + # Allow installing in editable mode for some packages that require it + # If set to the string "true" will install packages with `pip install -e .` + INSTALL_EDITABLE: ${{ vars.INSTALL_EDITABLE || 'false' }} + +defaults: + run: + shell: bash + +on: + # Triggers the workflow when pull request created and updated + pull_request: + # Allows run of this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + # Set variables that should be accessible to all jobs + set_variables: + runs-on: ${{ vars.RUNNER }} + env: + CURR_REPO: ${{ github.event.repository.name }} + outputs: + # The name of the untagged docker images to be created by this workflow + IMAGE_NAME: ${{ steps.set_image_name.outputs.IMAGE_NAME }} + # The name of the repository + REPO_NAME: ${{ steps.set_image_name.outputs.REPO_NAME }} + # The tag to use for the doclinttest images + DOCLINTTEST_TAG: ${{ steps.set_doclinttest_tag.outputs.DOCLINTTEST_TAG }} + # The image to use for doclinttest jobs + DOCLINTTEST_IMAGE: ${{ steps.set_doclinttest_image.outputs.DOCLINTTEST_IMAGE }} + # Flags for enabling/disabling various parts of the workflow + # Default to enabled unless overridden by a repository-level variable + CI_BUILD_SPHINX_HTML: ${{ steps.get_job_flags.outputs.CI_BUILD_SPHINX_HTML }} + CI_BUILD_SPHINX_PDF: ${{ steps.get_job_flags.outputs.CI_BUILD_SPHINX_PDF }} + CI_TEST_INTERFACES: ${{ steps.get_job_flags.outputs.CI_TEST_INTERFACES }} + CI_TEST_PYTEST_SHORT: ${{ steps.get_job_flags.outputs.CI_TEST_PYTEST_SHORT }} + steps: + - name: Set the image name + id: set_image_name + run: | + repo_name=$(echo ${{ github.repository }} | cut -d'/' -f2) + # Create an image name using the organiztion and repository names, lowercase + image_name=${{ vars.DOCKER_REGISTRY_USER }}/geoips + echo "REPO_NAME=${repo_name}" + echo "REPO_NAME=${repo_name}" >> ${GITHUB_ENV} + echo "REPO_NAME=${repo_name}" >> ${GITHUB_OUTPUT} + echo "IMAGE_NAME=${image_name}" + echo "IMAGE_NAME=${image_name}" >> ${GITHUB_ENV} + echo "IMAGE_NAME=${image_name}" >> ${GITHUB_OUTPUT} + - name: Set doclinttest image tag + id: set_doclinttest_tag + run: | + echo "CURR_REPO: ${CURR_REPO}" + # If the current repository is geoips, use the image we're about to build + # for running tests and building docs. + if [[ "${CURR_REPO}" == "geoips" ]]; then + echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" + echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" >> ${GITHUB_ENV} + echo "DOCLINTTEST_TAG=doclinttest-${GITHUB_SHA}" >> ${GITHUB_OUTPUT} + else + echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" + echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" >> ${GITHUB_ENV} + echo "DOCLINTTEST_TAG=${PLUGIN_DOCLINTTEST_TAG}" >> ${GITHUB_OUTPUT} + fi + - name: Set doclinttest image + id: set_doclinttest_image + run: | + owner=${{ github.repository_owner }} + owner=${owner,,} + doclinttest_image_name=${{ env.IMAGE_NAME }}:${DOCLINTTEST_TAG} + echo "DOCLINTTEST_IMAGE=${{ env.IMAGE_NAME }}:${DOCLINTTEST_TAG}" + echo "DOCLINTTEST_IMAGE=${{ env.IMAGE_NAME }}:${DOCLINTTEST_TAG}" >> ${GITHUB_ENV} + echo "DOCLINTTEST_IMAGE=${{ env.IMAGE_NAME }}:${DOCLINTTEST_TAG}" >> ${GITHUB_OUTPUT} + - name: Get job flags + id: get_job_flags + run: | + # Define an array of flags and their corresponding secret values + flags=(CI_BUILD_SPHINX_HTML \ + CI_BUILD_SPHINX_PDF \ + CI_TEST_INTERFACES \ + CI_TEST_PYTEST_SHORT) + for flag in "${flags[@]}"; do + # Dynamically construct the variable name and access the corresponding secret + case $flag in + CI_BUILD_SPHINX_HTML) val="${{ vars.CI_BUILD_SPHINX_HTML }}" ;; + CI_BUILD_SPHINX_PDF) val="${{ vars.CI_BUILD_SPHINX_PDF }}" ;; + CI_TEST_INTERFACES) val="${{ vars.CI_TEST_INTERFACES }}" ;; + CI_TEST_PYTEST_SHORT) val="${{ vars.CI_TEST_PYTEST_SHORT }}" ;; + esac + # Check if the secret value is "true" and set the environment variable and output + if [ "$val" == "false" ]; then + echo "${flag}=false" + echo "${flag}=false" >> $GITHUB_ENV + echo "${flag}=false" >> $GITHUB_OUTPUT + else + echo "${flag}=true" + echo "${flag}=true" >> $GITHUB_ENV + echo "${flag}=true" >> $GITHUB_OUTPUT + fi + done + + print_run_flags: + runs-on: ${{ vars.RUNNER }} + needs: [set_variables] + steps: + - name: Print run flags + run: | + echo "github.ref_name ${{ github.ref_name }}" + echo "github.ref ${{ github.ref }}" + echo "github.base_ref ${{ github.base_ref }}" + echo "github.head_ref ${{ github.head_ref }}" + echo "vars.RUNNER ${{ vars.RUNNER }}" + echo "vars.RUNNER_DOCKERBUILD ${{ vars.RUNNER_DOCKERBUILD }}" + echo "vars.RUNNER_DOCKERRUN ${{ vars.RUNNER_DOCKERRUN }}" + echo "github.event.repository.default_branch ${{ github.event.repository.default_branch }}" + echo "github.event.repository.name ${{ github.event.repository.name }}" + echo "github.event.pull_request.merged ${{ github.event.pull_request.merged }}" + echo "github.run_id ${{ github.run_id }}" + echo "github.run_attempt ${{ github.run_attempt }}" + echo "vars.USE_CONDA_FOR_ACTIONS ${{ vars.USE_CONDA_FOR_ACTIONS }}" + echo ${{ github.context }} + + for var in $(echo '${{ toJson(needs.set_variables.outputs) }}' | jq -r 'keys[]'); do + value=$(echo '${{ toJson(needs.set_variables.outputs) }}' | jq -r --arg var "$var" '.[$var]') + echo "$var=$value" + done + echo "Target ref:" + if [[ "${{ github.base_ref }}" != "" ]]; then + target_ref=${{ github.base_ref }} + else + target_ref=${{ github.event.repository.default_branch }} + fi + echo "remotes/origin/${target_ref}" + + # Build the base GeoIPS doclinttest image + # This image is used in all subsequent tests + # It is only built for the main GeoIPS package but is used by all repositories + # + # Within this image, the GeoIPS package is located in /packages/geoips and is + # installed using `pip install .[doc,lint,test]`. Scripts from geoips can be called + # from /packages/geoips. + build_doclinttest_image: + runs-on: ${{ vars.RUNNER_DOCKERBUILD || vars.RUNNER }} + needs: set_variables + if: > + always() && + vars.USE_CONDA_FOR_ACTIONS != 'true' && + github.event.repository.name == 'geoips' + env: + IMAGE_NAME: ${{ needs.set_variables.outputs.IMAGE_NAME }} + BUILDCACHE_IMAGE: ${{ needs.set_variables.outputs.IMAGE_NAME }}:buildcache + DOCLINTTEST_IMAGE: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} + # Can't write to registry without this + permissions: + contents: read + packages: write + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup BuildX + uses: docker/setup-buildx-action@v3 + + - name: Login to the Registry + uses: docker/login-action@v3 + with: + registry: ${{ vars.DOCKER_REGISTRY }} + username: ${{ vars.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + + # This cache allows the second push below when on the default branch + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Create tags + run: | + tags="${DOCLINTTEST_IMAGE},${BUILDCACHE_IMAGE}" + echo "GITHUB REF: ${GITHUB_REF}" + echo "Default Branch: refs/heads/${{ github.event.repository.default_branch }}" + echo "runner os: ${{ runner.os }}" + echo "github sha: ${{ github.sha }}" + echo "build cache image: ${{ env.BUILDCACHE_IMAGE }}" + if [[ "${GITHUB_REF}" == "refs/heads/${{ github.event.repository.default_branch }}" ]]; then + stableorlatest_image="${IMAGE_NAME}:doclinttest-stable" + tags="${tags},${stableorlatest_image}" + else + stableorlatest_image="${IMAGE_NAME}:doclinttest-latest" + tags="${tags},${stableorlatest_image}" + fi + # PUSH_TAGS is used with the docker buildx action + echo "PUSH_TAGS=${tags}" + echo "PUSH_TAGS=${tags}" >> $GITHUB_ENV + # STABLEORLATEST_IMAGE is only used with the explicit docker build command. + # It is the full path to the stable or latest image tag. + # The explicit docker build command requires pushing each image separately + # to the registry - so we need to track the stable/latest tag in addition + # to the DOCLINTTEST_IMAGE and BUILDCACHE_IMAGE tags + echo "STABLEORLATEST_IMAGE=${stableorlatest_image}" + echo "STABLEORLATEST_IMAGE=${stableorlatest_image}" >> $GITHUB_ENV + + # If using docker buildx was requested via organization variable, + # DO NOT run this section. + - name: Build and push docker image with explicit docker build + if: ${{ vars.USE_DOCKER_BUILDX_ACTION == 'false' }} + # > makes newlines into spaces, so single command on multiple lines. + run: > + which docker; + docker build + --file Dockerfile + --tag ${DOCLINTTEST_IMAGE} + --tag ${BUILDCACHE_IMAGE} + --tag ${STABLEORLATEST_IMAGE} + --target doclinttest + .; + docker push ${BUILDCACHE_IMAGE}; + docker push ${DOCLINTTEST_IMAGE}; + docker push ${STABLEORLATEST_IMAGE}; + - name: Build and push docker image with docker buildx action + if: ${{ vars.USE_DOCKER_BUILDX_ACTION != 'false' }} + uses: docker/build-push-action@v6 + with: + context: . + target: doclinttest + push: true + tags: ${{ env.PUSH_TAGS }} + file: "Dockerfile" + cache-from: type=registry,ref=${{ env.BUILDCACHE_IMAGE }} + cache-to: type=registry,ref=${{ vars.DOCKER_REGISTRY }}/${{ env.BUILDCACHE_IMAGE }},mode=max + + ################################################# + # Documentation builds - requires pip installed geoips, so use container + ################################################# + build_sphinx_html: + if: > + always() && + vars.USE_CONDA_FOR_ACTIONS != 'true' && + needs.set_variables.outputs.CI_BUILD_SPHINX_HTML == 'true' && + (needs.build_doclinttest_image.result == 'success' || + needs.build_doclinttest_image.result == 'skipped') + runs-on: ${{ vars.RUNNER_DOCKERRUN || vars.RUNNER }} + needs: [build_doclinttest_image, set_variables] + container: + image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} + credentials: + username: ${{ vars.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + env: + CURR_REPO: ${{ github.event.repository.name }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure Git safe directory + run: git config --global --add safe.directory $PWD + + - name: Install package + run: | + if [ "${INSTALL_EDITABLE}" = "true" ]; then + pip install -e . + else + pip install . + fi + + - name: Run build html + id: build-html-docs + run: | + # If we are specifically "template_basic_plugin" repo, then we must use + # my_package as the package name. Otherwise the package name matches + # the plugin name. + run_on_package=$CURR_REPO + if [[ "$CURR_REPO" == "template_basic_plugin" ]]; then + run_on_package=my_package + fi + echo "Build html docs" + # Call build_docs.sh from the geoips default branch + # Pass in + # the path to the current plugin repo, + # the name of the current repo, + # "html_only" to indicate producing only html output, and + # the path to the docs template directory in the geoips default branch. + $BASE_GEOIPS_PATH/docs/build_docs.sh . $run_on_package html_only $BASE_GEOIPS_PATH/docs + ret=$? + if [[ "${ret##*:}" != *"0"* ]]; then + echo "::error::Building html docs ${ret##*:}" + exit 1 + fi + echo "BUILT_DOCS_PATH=$PWD/build/sphinx/html" >> $GITHUB_OUTPUT + - name: Upload HTML Docs + if: ${{ vars.UPLOAD_GITHUB_ARTIFACTS == 'true' }} + uses: actions/upload-artifact@v4 + with: + name: HTML Docs + path: ${{ steps.build-html-docs.outputs.BUILT_DOCS_PATH }} + if-no-files-found: error + + # This job does not currently work because latex is not installed in the image. I think + # that to get this job working would only require installing texlive-base but other + # texlive packages might also be required. + # + # Additionally, this job is extremely slow. Building the PDF documentation is time + # consuming and should probably only be run on releases. Building the HTML documentation + # should be enough to catch any problems that crop up. + # + # build_sphinx_pdf: + # if: ${{ needs.set_variables.outputs.CI_DISABLE_BUILD_SPHINX_PDF == 'false' }} + # runs-on: ${{ vars.RUNNER_DOCKERRUN || vars.RUNNER }} + # needs: [build_doclinttest_image, set_variables] + # container: + # image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} + # credentials: + # username: ${{ secrets.DOCKER_REGISTRY_USER }} + # password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + # env: + # CURR_REPO: ${{ github.event.repository.name }} + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + # + # - name: Configure Git safe directory + # run: git config --global --add safe.directory $PWD + # + # - name: Install geoips documentation packages + # run: pip install -e .[doc] + # + # - name: Run build pdf + # run: | + # echo "Build pdf docs" + # # Call build_docs.sh from the geoips default branch + # # Pass in + # # the path to the current plugin repo, + # # the name of the current repo, + # # "pdf_only" to indicate producing only pdf output, and + # # the path to the docs template directory in the geoips default branch. + # ./docs/build_docs.sh . $CURR_REPO pdf_only ./docs + # ret=$? + # if [[ "${ret##*:}" != *"0"* ]]; then + # echo "::error::Building pdf docs ${ret##*:}" + # exit 1 + # fi + + ################################################# + # Code tests - requires pip installed geoips, so use container + ################################################# + test_interfaces: + if: > + always() && + vars.USE_CONDA_FOR_ACTIONS != 'true' && + needs.set_variables.outputs.CI_TEST_INTERFACES == 'true' && + (needs.build_doclinttest_image.result == 'success' || + needs.build_doclinttest_image.result == 'skipped') + runs-on: ${{ vars.RUNNER_DOCKERRUN || vars.RUNNER }} + needs: [build_doclinttest_image, set_variables] + container: + image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} + credentials: + username: ${{ vars.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Configure Git safe directory + run: git config --global --add safe.directory $PWD + - name: Pip install package + run: pip install -v . + - name: create_plugin_registries + run: create_plugin_registries + - name: Run code check script interfaces + run: | + $BASE_GEOIPS_PATH/tests/utils/check_code.sh interfaces geoips + ret=$? + echo "::group::interface_analysis" + echo "INTERFACES analysis of code" + echo "${ret}" + echo "::endgroup::" + if [[ "${ret##*:}" != *"0"* ]]; then + echo "::error::due to interface violations ${ret##*:}" + exit 1 + fi + + test_pytest_short: + if: > + always() && + vars.USE_CONDA_FOR_ACTIONS != 'true' && + needs.set_variables.outputs.CI_TEST_PYTEST_SHORT == 'true' && + (needs.build_doclinttest_image.result == 'success' || + needs.build_doclinttest_image.result == 'skipped') + runs-on: ${{ vars.RUNNER_DOCKERRUN || vars.RUNNER }} + needs: [build_doclinttest_image, set_variables] + container: + image: ${{ needs.set_variables.outputs.DOCLINTTEST_IMAGE }} + credentials: + username: ${{ vars.DOCKER_REGISTRY_USER }} + password: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + env: + CURR_REPO: ${{ github.event.repository.name }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Configure Git safe directory + run: git config --global --add safe.directory $PWD + - name: Pip install package + run: pip install -v . + - name: create_plugin_registries + run: create_plugin_registries + # Run unit tests for this repo if they exist + - name: Run pytest unit tests + run: | + echo "::group::pytest_unit_test" + echo "Pytest short unit tests of ${CURR_REPO}" + echo "which pytest" + which pytest + if [ ! -d "./tests/unit_tests" ]; then + echo "No unit tests found in ${CURR_REPO}" + exit 0 + fi + echo "pytest -q ./tests/unit_tests" + pytest -q ./tests/unit_tests + ret=$? + echo "Return code: ${ret}" + echo "::endgroup::" + if [[ "${ret}" != *"0"* ]]; then + echo "::error::due to geoips repo pytest errors, return code ${ret}" + exit 1 + fi + - name: Run base GeoIPS pytest unit tests + if: ${{ env.CURR_REPO != 'geoips' }} + run: | + echo "::group::pytest_unit_test" + echo "Pytest short unit tests of ${CURR_REPO} using core GeoIPS tests" + echo "which pytest" + which pytest + echo "pytest -q $BASE_GEOIPS_PATH/tests/unit_tests" + pytest -q $BASE_GEOIPS_PATH/tests/unit_tests + ret=$? + echo "Return code: ${ret}" + echo "::endgroup::" + if [[ "${ret}" != *"0"* ]]; then + echo "::error::due to geoips repo pytest errors, return code ${ret}" + exit 1 + fi diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 000000000..dc6066e2f --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,18 @@ +name: Lint + +# Note we want to run linting all the time +on: + # Triggers the workflow when pull request created and updated + pull_request: + # Allows run of this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + lint: + name: Lint + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-lint.yaml@main + permissions: + contents: read + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/package-and-publish.yaml b/.github/workflows/package-and-publish.yaml index 33c33a6a0..033274226 100644 --- a/.github/workflows/package-and-publish.yaml +++ b/.github/workflows/package-and-publish.yaml @@ -1,31 +1,38 @@ -name: Package and publish +name: Package/publish + +# 1. Brassy - on merge of v*-version-release +# * generate release notes +# * commit and push to v*-add-rst-release-note +# * open PR to default branch +# 2. Tag and Release - on merge of v*-add-rst-release-note +# * Tag current version +# * Release just tagged version +# 3. Package and Publish - on published release (from #2) +# * Build wheel +# * Publish to pypi +# 4. Deploy docs - on published release (from #2) +# * pip install geoips +# * pip install plugin repo +# * build docs with geoips/docs/build_docs.sh +# * deploy docs with geoips/docs/deploy_pages.sh on: - workflow_dispatch: + # triggers the workflow on published release release: - types: [published] + types: + - published + # allows run of this workflow manually from the actions tab + # must be merged to default before it will be available to manually run. + workflow_dispatch: jobs: - deploysdist: - name: "Deploy to Pypi" - runs-on: ${{ vars.RUNNER }} - - steps: - - name: Checkout source - uses: actions/checkout@v3 - with: - fetch-depth: 0 - fetch-tags: true - - - name: Create distributions - shell: bash -l {0} - run: | - python -m pip install -U build pip - python -m build - - name: Publish package to PyPI - # upload to PyPI on every release for a tag starting with 'v' - # if: github.event.action == 'published' && startsWith(github.event.release.tag_name, 'v') - uses: pypa/gh-action-pypi-publish@v1.8.10 - with: - user: ${{ secrets.PYPI_USER }} - password: ${{ secrets.PYPI_PASSWORD }} + package-publish: + name: Package/publish + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-package-and-publish.yaml@main + permissions: + contents: read + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + pypi_user: ${{ secrets.PYPI_USER }} + pypi_password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/proper-release-note-edits.yaml b/.github/workflows/proper-release-note-edits.yaml new file mode 100644 index 000000000..9b08747d4 --- /dev/null +++ b/.github/workflows/proper-release-note-edits.yaml @@ -0,0 +1,16 @@ +name: Note edits + +# We only need to check status of release notes from pull requests. +on: + # Triggers the workflow when pull request created and updated + pull_request: + # Allows run of this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + note-edits: + name: Note edits + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-proper-release-note-edits.yaml@main + permissions: + contents: read diff --git a/.github/workflows/tag-and-release.yaml b/.github/workflows/tag-and-release.yaml new file mode 100644 index 000000000..f8a1e2399 --- /dev/null +++ b/.github/workflows/tag-and-release.yaml @@ -0,0 +1,25 @@ +name: Tag/release + +on: + # Triggers the workflow when pull request closed (either closed + # as non-planned, or actually approved and merged.) + pull_request: + types: + - closed + branch: main + # Allows run of this workflow manually from the Actions tab + # Must be merged to default branch before it will be available + # to manually run. + workflow_dispatch: + +jobs: + tag-release: + name: Tag/release + # You do not appear to be able to use variables in the "uses" field. + uses: NRLMMD-GEOIPS/geoips_ci/.github/workflows/reusable-tag-and-release.yaml@main + # Only run this if the pull request was merged, not just closed. + if: github.event.pull_request.merged == true + permissions: + contents: write + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows_archived/on-closed-pr.yaml b/.github/workflows_archived/on-closed-pr.yaml new file mode 100644 index 000000000..48d938bdc --- /dev/null +++ b/.github/workflows_archived/on-closed-pr.yaml @@ -0,0 +1,57 @@ +name: Call GeoIPS workflows triggered on PR close + +on: + # Triggers the workflow when pull request created and updated + pull_request: + workflow_dispatch: + +jobs: + print_run_flags: + runs-on: ${{ vars.RUNNER }} + steps: + - name: Print run flags + run: | + echo "github.ref_name ${{ github.ref_name }}" + echo "github.ref ${{ github.ref }}" + echo "github.base_ref ${{ github.base_ref }}" + echo "github.head_ref ${{ github.head_ref }}" + echo "vars.RUNNER ${{ vars.RUNNER }}" + echo "vars.RUNNER_DOCKERBUILD ${{ vars.RUNNER_DOCKERBUILD }}" + echo "vars.RUNNER_DOCKERRUN ${{ vars.RUNNER_DOCKERRUN }}" + echo "github.event.repository.default_branch ${{ github.event.repository.default_branch }}" + echo "github.event.repository.name ${{ github.event.repository.name }}" + echo "github.event.pull_request.merged ${{ github.event.pull_request.merged }}" + echo "github.run_id ${{ github.run_id }}" + echo "github.run_attempt ${{ github.run_attempt }}" + echo "vars.USE_CONDA_FOR_ACTIONS ${{ vars.USE_CONDA_FOR_ACTIONS }}" + echo "$GITHUB_CONTEXT" + echo ${{ github.context }} + + set_variables: + runs-on: ${{ vars.RUNNER }} + env: + CURR_REPO: ${{ github.event.repository.name }} + outputs: + # The name of the untagged docker images to be created by this workflow + pr_was_merged: ${{ steps.set_pr_status.outputs.PR_WAS_MERGED }} + steps: + - name: Set the PR status + id: set_pr_status + run: | + pr_was_merged=${{ github.event.pull_request.merged }} + echo "$pr_was_merged" + echo "PR_WAS_MERGED=$pr_was_merged" + echo "PR_WAS_MERGED=$pr_was_merged" >> $GITHUB_ENV + echo "PR_WAS_MERGED=$pr_was_merged" >> $GITHUB_OUTPUT + + call_on_closed_pr: + permissions: + pull-requests: write + contents: write + # You do not appear to be able to use variables in this "uses" field. + uses: NRLMMD-GEOIPS/.github/.github/workflows/on-closed-pr.yaml@main + needs: set_variables + with: + pr_was_merged: ${{ needs.set_variables.outputs.PR_WAS_MERGED }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 11ac4401d..2f690a1f6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,10 @@ profiling/ geoips/commandline/ancillary_info/cmd_instructions.json # auto-generated alias mapping file for the cli geoips/commandline/ancillary_info/alias_mapping.json -# auto-generated release note file for latest changes -docs/source/releases/latest.rst +# auto-generated release note files, all release note information +# is stored in .yaml format and the rst files auto-generated. +# index.rst, latest.rst, v*.rst +docs/source/releases/*.rst # auto-generated plugin registry file registered_plugins.json # auto-generated version file, created at run-time diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 726720a79..93be05f54 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,11 +7,3 @@ Please see https://github.com/NRLMMD-GEOIPS/geoips/blob/main/CHANGELOG_TEMPLATE.rst for instructions on updating release notes appropriately with each PR. - -The release note that is currently, actively being updated in -all geoips plugin repositories is referenced in the file: - -* https://github.com/NRLMMD-GEOIPS/geoips/blob/main/update_this_release_note - -* Please add all release notes for the current PR in the file referenced - within ``update_this_release note``. diff --git a/CHANGELOG_TEMPLATE.rst b/CHANGELOG_TEMPLATE.rst index 230033429..136719a65 100644 --- a/CHANGELOG_TEMPLATE.rst +++ b/CHANGELOG_TEMPLATE.rst @@ -3,338 +3,20 @@ | # # # This source code is protected under the license referenced at | # # # https://github.com/NRLMMD-GEOIPS. -:: - - #################################################################################### - DO NOT MODIFY THIS TEMPLATE! - FOR USE IN COMPILING CONSISTENT RELEASE NOTES! - - NOTE: RST IS VERY PARTICULAR ABOUT SPACING / INDENTATIONS! - FOLLOW THIS TEMPLATE CLOSELY TO ENSURE PROPER RENDERING (and read up on rst syntax) - #################################################################################### - -1. Follow below template when adding information to the current release note - prior to opening a pull request - - - Follow proper formatting/categorization of changes below - - - newest changes go at the top, just below distribution - statement. - - Ensure sections are in correct order, as listed below - - Follow spacing and indentation in template closely, - RST is very particular about spacing and indentation - (and read up on RST syntax for good measure). - - - If you make changes during the Pull Request process, just update - the release note entry accordingly, do not add an additional - release note entry. We only want the information regarding the final - merged pull request in the release notes. - -2. Only include headers for categories that have current updates (do NOT - include empty headers - if there are no "Breaking Changes" do not include - the "Breaking Changes" header) - -3. Include a list of files for each "Summary of change" section. - - - Simple changes can just have the list of modified files directly - under the “Summary of change” section, without additional details. - -4. Issue ID should match related linked issue, like: NRLMMD-GEOIPS/geoips#27 -5. Ensure your entries are included directly in the release note file: - docs/source/releases/vX_Y_Z.rst - - - NOTE: X.Y.Z is the UPCOMING VERSION of the repository, NOT the current - tagged version! - - The current release note to update directly on any geoips plugin - repository with current changes is referenced in the file - `update_this_release_note` within the geoips repo default branch: - https://github.com/NRLMMD-GEOIPS/geoips/blob/main/update_this_release_note - -:: - - ################################################################################### - NOTE: RST IS VERY PARTICULAR ABOUT SPACING / INDENTATIONS! - FOLLOW THIS TEMPLATE CLOSELY TO ENSURE PROPER RENDERING (and read up on rst syntax) - ################################################################################### - - -Version X.Y.Z (YYYY-MM-DD) -************************** - -* Very brief bulleted summary of changes -* Should reference major changes outlined below - -Breaking Changes -================ - - -------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* More info about the change - - * Additional optional information - -:: - - modified: /path/to/changed/file/related/to/breaking/change/1 - new: /path/to/new/file/related/to/breaking/change/1 - deleted: /path/to/deleted/file/related/to/breaking/change/1 - renamed: /path/to/orig/file -> /path/to/new/file - - -------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* More info about breaking Change 2 - - * Additional optional information - -* More info about breaking change 2 - -:: - - modified: /path/to/changed/file1/related/to/breaking/change/2 - modified: /path/to/changed/file2/related/to/breaking/change/2 - -Security Updates -================ - - ---------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* required security change due to file update - -:: - - modified: /paths/to/changed/file(s) - - -Major New Functionality -======================= - - -------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to new functionality - -:: - - modified: /paths/to/changed/file(s) - - -Enhancements -============ - - ------------------------------------------------ - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to improvements or enhancements to the code - -:: - - modified: /paths/to/changed/file(s) - - -Deprecations -============ - - ------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to code that has been or will be deprecated - -:: - - modified: /paths/to/changed/file(s) - - -Regression Fixes -================ - - ------------------------------------------------------------ - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to code that used to work in a previous release, - and is now broken. - -:: - - modified: /paths/to/changed/file(s) - - -Bug Fixes -========= - - ------------------------------------------ - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to bug fixes - -:: - - modified: /paths/to/changed/file(s) - - -Efficiency Improvements -======================= - - ------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to efficiency improvements - -:: - - modified: /paths/to/changed/file(s) - - -Installation Updates -==================== - - --------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to installation updates - -:: - - modified: /paths/to/changed/file(s) - - -Real-time Processing Updates -============================ - - ---------------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to real-time processing updates - -:: - - modified: /paths/to/changed/file(s) - - -Refactoring Updates -=================== - - ------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to code refactoring - -:: - - modified: /paths/to/changed/file(s) - - -Code Formatting and Style Updates -================================= - - ------------------------------------------------------------ - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to code formatting and style updates - -:: - - modified: /paths/to/changed/file(s) - - -GitHub Actions Updates -====================== - - ---------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to GitHub Actions updates - -:: - - modified: /paths/to/changed/file(s) - - -Git Workflow Improvements -========================= - - -------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change related to Git workflow improvements - -:: - - modified: /paths/to/changed/file(s) - -Test Repo Updates -================= - - --------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change to TEST REPO OUTPUT - -:: - - modified: /paths/to/changed/file(s) - -Testing Updates -=============== - - --------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change that WILL IMPACT TESTING - (or change to testing process) - -:: - - modified: /paths/to/changed/file(s) - - -Documentation Updates -===================== - - --------------------------------------------------------------- - -*From NRLMMD-GEOIPS/geoips#NN: YYYY-MM-DD, * - -* Details about change to documentation (contents, or build process) - -:: - - modified: /paths/to/changed/file(s) - +The GeoIPS Ecosystem uses ``brassy`` for automating RST release note generation from +user-generated YAML release notes. Every pull request within the GeoIPS GitHub +organization must have valid YAML release notes in the ``docs/source/releases/latest/`` +directory prior to being approved and merged. + +1. Run `brassy -t docs/source/releases/latest/.yaml` to create a template + release note that you can modify. +2. Edit brassy YAML release note template with all updates made on current branch. + + * Select appropriate category as found in the template, or create your own! + + * Empty sections in the YAML template can remain - they will be ignored. + * **Title** should be a short description of the change, < 120 characters, preferably <60 + * **Description** can be multiple lines (use proper YAML formatting), with additional + details. + * Always include a list of add/modified/deleted **files** for each entry. + * **Related Issue** ID should match related linked issue, like: NRLMMD-GEOIPS/geoips#27 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 0f6f2d247..bb34ea19f 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,3 +1,6 @@ + # # # This source code is protected under the license referenced at + # # # https://github.com/NRLMMD-GEOIPS. + # Contributor Covenant Code of Conduct This Code of Conduct is maintained and enforced by our community collaborators, diff --git a/Dockerfile b/Dockerfile index 7f3699f78..b514d071f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-bullseye as base +FROM python:3.10-slim-bullseye AS base RUN apt-get update && apt-get -y upgrade \ && apt-get install -y wget git libopenblas-dev \ diff --git a/LICENSE b/LICENSE index 92411ba4b..9ad9ee4bf 100644 --- a/LICENSE +++ b/LICENSE @@ -9,6 +9,7 @@ rights or licenses to the Computer Software are granted. Unauthorized use, sale, conveyance, disposition, or modification of a Computer Software may result in civil penalties and/or criminal penalties under 18 U.S.C. § 641. + 2. For purposes of this Agreement, the following definitions apply: a. “Computer software” means computer programs, source code, source code diff --git a/bandit.yml b/bandit.yml new file mode 100644 index 000000000..c316793bf --- /dev/null +++ b/bandit.yml @@ -0,0 +1,2 @@ +exclude: + - build/ diff --git a/docs/build_docs.sh b/docs/build_docs.sh index 0320b12e8..1a4809f66 100755 --- a/docs/build_docs.sh +++ b/docs/build_docs.sh @@ -105,12 +105,6 @@ if [[ "$4" != "" ]]; then else geoipsdocpath="$GEOIPS_PACKAGES_DIR/geoips/docs" fi -if [[ "$5" != "" ]]; then - echo "passed geoips_vers=$5" - geoips_vers=$5 -else - geoips_vers=latest -fi if [[ ! -d "$geoipsdocpath" ]]; then echo "***************************************************************************" echo "ERROR: GeoIPS docs path, with templates, does not exist:" @@ -166,54 +160,97 @@ fi # Since we revert index.rst at the end of this script, make sure the # user does not have any local modifications before starting. git_status_index=`git -C $docbasepath status docs/source/releases/index.rst` -if [[ "$git_status_index" == *"docs/source/releases/index.rst"* ]]; then +if [[ "$git_status_index" == *"docs/source/releases/"*".rst"* ]]; then echo "***************************************************************************" - echo "ERROR: Do not modify docs/source/releases/index.rst directly" - echo "Auto-generated within build_docs.sh using brassy." + echo "ERROR: Do not modify docs/source/releases/*.rst directly" + echo "All RST release files are auto-generated within build_docs.sh using brassy." echo "Please revert your changes and try again." echo "***************************************************************************" exit 1 fi -# Release notes are ALWAYS written in the "latest" folder, whether we are -# producing the generic "latest.rst" release note, or the specific -# vX_Y_Z.rst release note for an actual version release. -current_release_notes=`ls $docbasepath/docs/source/releases/latest/*` -# If we found any current YAML release notes, we need to generate the -# release RST file, and update index.rst with the current release version. -if [[ "$current_release_notes" != "" ]]; then - echo "Running brassy to generate current release note ${geoips_vers}.rst" +# Loop through all of the directories in docs/source/releases. +# Each set of YAML release notes is in a single directory, so +# we will build them all into RST release notes prior to building +# the docs. +for version_dir in $docbasepath/docs/source/releases/*/ ; do + echo $version_dir + # Only attempt to build release notes from a directory if there + # are YAML release notes in the directory. + notes_in_dir=`ls $version_dir/*.yaml` + if [[ "$notes_in_dir" == "" ]]; then + echo "SKIPPING: Empty directory $version_dir, skipping" + continue + fi + # Remove the trailing '/' from the version_dir (the way the + # directories were listed with /*/ above the trailing '/' + # is included in the path) + version=`basename ${version_dir::-1}` + # Explicitly skip the "upcoming" folder in the documentation. + # By the time we are ready to include these YAML release notes, + # they will be moved to the "latest" directory. "upcoming" only + # exists during the release process. + if [[ "$version" == "upcoming" ]]; then + echo "SKIPPING: Not including 'upcoming' version in documentation." + continue + fi + + # Create a temporary file with the header for the release note. + # This is passed into brassy as the prefix-file. + RELEASE_NOTE_HEADER=$docbasepath/docs/source/releases/RELEASE_NOTE_HEADER + touch $RELEASE_NOTE_HEADER + echo ".. dropdown:: Distribution Statement" >> $RELEASE_NOTE_HEADER + echo " " >> $RELEASE_NOTE_HEADER + echo " | This file is auto-generated. Please abide by the license found at" \ + >> $RELEASE_NOTE_HEADER + echo " | ${GEOIPS_REPO_URL}." \ + >> $RELEASE_NOTE_HEADER + + echo "Running brassy to generate current release note ${version}.rst" echo "" - echo "touch $docbasepath/docs/source/releases/${geoips_vers}.rst" - echo "brassy --release-version $geoips_vers --no-rich \" - echo " --output-file $docbasepath/docs/source/releases/${geoips_vers}.rst \" - echo " $docbasepath/docs/source/releases/latest" + echo "touch $docbasepath/docs/source/releases/${version}.rst" + echo "brassy --release-version $version --no-rich \ " + echo " --prefix-file $RELEASE_NOTE_HEADER \ " + echo " --output-file $docbasepath/docs/source/releases/${version}.rst \ " + echo " $docbasepath/docs/source/releases/$version" + echo "pink $docbasepath/docs/source/releases/${version}.rst" echo "" # Right now brassy does not auto-generate vers.rst, so we must touch it in # advance. - touch $docbasepath/docs/source/releases/${geoips_vers}.rst + touch $docbasepath/docs/source/releases/${version}.rst # Brassy creates the rst file! - brassy --release-version $geoips_vers --no-rich \ - --output-file $docbasepath/docs/source/releases/${geoips_vers}.rst \ - $docbasepath/docs/source/releases/latest - if [[ "$?" != "0" ]]; then + brassy --release-version $version --no-rich \ + --prefix-file $RELEASE_NOTE_HEADER \ + --output-file $docbasepath/docs/source/releases/${version}.rst \ + $docbasepath/docs/source/releases/${version} + brassy_retval=$? + # Remove the temporary release note header + rm -f $docbasepath/docs/source/releases/RELEASE_NOTE_HEADER + + # Pinken the RST file + pink $docbasepath/docs/source/releases/${version}.rst + pink_retval=$? + if [[ "$brassy_retval" != "0" || "$pink_retval" != "0" ]]; then # Exit here if brassy failed, because the doc build will subsequently fail. - echo "FAILED brassy ${geoips_vers}.rst release note generation failed." - echo "Please resolve release note formatting noted above and retry" - exit 1 - fi - # Ensure index.rst is updated with the latest release notes. - # This will eventually likely be rolled into brassy. - echo "Adding latest section to release note index" - echo "python $geoipsdocpath/update_release_note_index.py $docbasepath/docs/source/releases/index.rst $geoips_vers" - python $geoipsdocpath/update_release_note_index.py $docbasepath/docs/source/releases/index.rst $geoips_vers - if [[ "$?" != "0" ]]; then - # If this failed, exit here, because doc build will subsequently fail. - echo "FAILED update_release_note_index.py for version ${geoips_vers}" + echo "FAILED brassy ${version}.rst release note generation failed." echo "Please resolve release note formatting noted above and retry" + echo "brassy retval: $brassy_retval" + echo "pink retval: $pink_retval" exit 1 fi +done + +# Ensure index.rst is updated with the latest release notes. +# This will eventually likely be rolled into brassy. +echo "Creating docs/source/releases/index.rst with all release notes" +echo "python $geoipsdocpath/update_release_note_index.py $docbasepath/docs/source/releases/index.rst $docbasepath/docs/source/releases" +python $geoipsdocpath/update_release_note_index.py $docbasepath/docs/source/releases/index.rst $docbasepath/docs/source/releases +if [[ "$?" != "0" ]]; then + # If this failed, exit here, because doc build will subsequently fail. + echo "FAILED update_release_note_index.py." + echo "Please resolve release note formatting noted above and retry" + exit 1 fi buildfrom_docpath=$docbasepath/build/buildfrom_docs @@ -237,9 +274,9 @@ fi # below, and in that explicit order. # (directory name within docs/source, followed by heading name within index.rst # in parentheses below): -# * Required: introduction (Introduction) +# * Optional: introduction (Introduction) # * Optional: starter (Getting Started) -# * Required: userguide (User Guide) +# * Optional: userguide (User Guide) # * Optional: devguide (Developer Guide) # * Optional: deployment_guide (Deployment Guide) - note, NOT in geoips repo # * Optional: operator_guide (Operator Guide) - note, NOT in geoips repo @@ -249,24 +286,36 @@ fi # Plugin package documentation will follow the same order, only including the # sections included in their docs/source directory. -# Note api, introduction, and userguide are required for all plugin packages. +# Note api and releases are required for all plugin packages. # Those are hard coded in source/_templates/index_PKG.html -# starter, devguide, and contact are optional - if index.rst exist for those -# sections within a plugin package, include them in the main index. If they -# do not exist, don't include them. They are added to index_PKG.html template +# introduction, starter, userguide, devguide, and contact are optional - +# if index.rst exist for those sections within a plugin package, include +# them in the main index. If they do not exist, don't include them. +# They are added to index_PKG.html template # by search/replacing for *IDX in the template below. + +# API and releases are NOT included in these search/replace templates, +# since they are hard coded in index_PKG.html, so are required to be included. +introidx="introduction\/index" startidx="starter\/index" +userguideidx="userguide\/index" devguideidx="devguide\/index" deployguideidx="deployguide\/index" opguideidx="opguide\/index" contactidx="contact\/index" # Only include the following links in the index if they exist +if [[ ! -f $docbasepath/docs/source/introduction/index.rst ]]; then + introidx="" +fi if [[ ! -f $docbasepath/docs/source/deployguide/index.rst ]]; then deployguideidx="" fi if [[ ! -f $docbasepath/docs/source/opguide/index.rst ]]; then opguideidx="" fi +if [[ ! -f $docbasepath/docs/source/userguide/index.rst ]]; then + userguideidx="" +fi if [[ ! -f $docbasepath/docs/source/starter/index.rst ]]; then startidx="" fi @@ -323,15 +372,14 @@ if [[ "$pdf_required" == "True" ]]; then echo " try 'conda install latexcodec' if in anaconda" echo " or re-run with html_only to only build" echo " html documentation." - echo "" - echo "Reverting $docbasepath/docs/source/releases/index.rst" - git -C $docbasepath checkout docs/source/releases/index.rst exit 1 fi # do not include release notes in the PDF # Include starter, contact, devguide if they exist. releasidx="" sed "s/PKGNAME/${pkgname}/g; s/STARTERIDX/${startidx}/g; \ + s/INTROIDX/${introidx}/g; \ + s/USERGUIDEIDX/${userguideidx}/g; \ s/CONTACTIDX/${contactidx}/g; \ s/DEPLOYGUIDEIDX/${deployguideidx}/g; s/OPGUIDEIDX/${opguideidx}/g; \ s/DEVIDX/${devguideidx}/g; s/RELEASEIDX/${releasidx}/g;" \ @@ -385,6 +433,8 @@ if [[ "$html_required" == "True" ]]; then releasidx="releases\/index" # Include starter, contact, devguide if they exist. sed "s/PKGNAME/${pkgname}/g; s/STARTERIDX/${startidx}/g; \ + s/INTROIDX/${introidx}/g; \ + s/USERGUIDEIDX/${userguideidx}/g; \ s/CONTACTIDX/${contactidx}/g; \ s/DEPLOYGUIDEIDX/${deployguideidx}/g; s/OPGUIDEIDX/${opguideidx}/g; \ s/DEVIDX/${devguideidx}/g; s/RELEASEIDX/${releasidx}/g;" \ @@ -420,13 +470,6 @@ fi echo "" echo "***" -# docs/source/releases/index.rst should only be auto-generated, -# so revert the changes we just made. Note we checked at the beginning -# if this was already modified, and exited if there were any local modifications, -# to ensure the user had not manually modified it. -# git -C $docbasepath status docs/source/releases/index.rst -echo "Reverting $docbasepath/docs/source/releases/index.rst" -git -C $docbasepath checkout docs/source/releases/index.rst date -u echo "***" echo "" diff --git a/docs/dev/coding_standards.rst b/docs/dev/coding_standards.rst new file mode 100644 index 000000000..d77db74be --- /dev/null +++ b/docs/dev/coding_standards.rst @@ -0,0 +1,205 @@ +.. dropdown:: Distribution Statement + + | # # # This source code is protected under the license referenced at + | # # # https://github.com/NRLMMD-GEOIPS. + +================================= +🖍️ Coding Standards Reference Doc +================================= + +*"Although that way may not be obvious at first unless you're Dutch."* +- the Zen of Python + + +.. contents:: + +Summary +======= + +The goal is consistency. In style, we're in the right even if we're all wrong together. +GeoIPS code is meant to be re-used by many, and we expect that the code will be read +much more than it is written. The goal of this document is to help us spend less time +on conventions and more time making GeoIPS better. + +We use - in order of supremacy - a few Internal Standards, followed by the numpy +docstring standards and the python style guide laid out in PEP 8. + +This document will detail internal standard conventions and include some of the external +standards for easy reference. For external standards, the primary source is always +right in the case of a conflict with this document. + +External Style Standards +------------------------ + +`PEP 8 `__ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A nice version of PEP8 can be found at: `PEP8.org ` + +Some highlights below for reference. + +Function Names +"""""""""""""" + +Function names should be lowercase, with words separated by +underscores as necessary to improve readability. + +`PEP8 Names Standards `__ + +Class Names +""""""""""" + +Class names should normally use the CapWords convention. + +`PEP8 Names Standards `__ + +Exception Names +""""""""""""""" + +Because exceptions should be classes, the class naming convention applies here. +However, you should use the suffix "Error" on your exception names +(if the exception actually is an error). + +`PEP8 Names Standards `__ + +Module Names +"""""""""""" + +Modules should have **short**, **all-lowercase names**. +Underscores can be used in the module name if it improves readability. +`PEP8 Names Standards `__ + +Imports +^^^^^^^ +Imports should usually be on separate lines, e.g.: + +Yes: + +.. code-block:: python + + import os + import sys + +No: + +.. code-block:: python + + import os, sys + +It's okay to say this though: + +.. code-block:: python + + from subprocess import Popen, PIPE + +Imports are always put at the top of the file, just after any module comments and +docstrings, and before module globals and constants. + +`PEP8 Imports Standards `__ + +Internal Style Standards +------------------------- + +Bring code to standard in a dedicated PR +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We want to separate formatting/standardizing and functional changes to the code so +reviewing code is less painful. Please, if you're improving the functionality of code +and need to bring it to standard: + +1. Make a new branch (branch1) +2. Bring the code to standard +3. Open a PR and make a new branch from branch1 (branch2) +4. Make improvements to the functionality of the code on branch 2 +5. Open a second PR for branch 2 + +If easier, you can make the improvement before bringing the code to standard. + +If you touch code, it should meet standards +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We strongly recommend you update any functions you work on +if they do not meet the standard. At CIRA, this is a requirement for +PRs to be merged. For others, it's just a strong recommendation - +however, we don't want the burden of updating code to prevent you from contributing. +Please don't spend hours updating a 100,000 line module because you fixed a typo. +use discretion on when updates are needed. + +A good rule of thumb is that if you edit something and it doesn't have a docstring, +add it. If you edit more than 20% of a function/class/module, please edit the rest. + +Imports shouldn't be buried without a reason +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If an import needs to be buried for efficiency reasons or namespace conflicts, +this should be documented in the docstrings. + + +Linting/Formatting +^^^^^^^^^^^^^^^^^^ + +The GeoIPS project makes use of several linting tools to help maintain code quality. The +full suite of linters can be installed by installing the "test" dependencies via pip. +For example, if you installed GeoIPS using `pip install .` the linters can be installed +using `pip install .[test]` the following tools to ensure code quality: + +Black +^^^^^ + +We use the `Black formatter `_ with its default +settings. As stated in the Black documentation, it is an uncompromizing code +formatter, but it has resulted in significantly more readable code. Applying it +automatically while writing code has also reduced development time since +developers don't need to think about formatting. + +Flake8 +^^^^^^ + +We use the `Flake8 linter `_ to enforce +PEP8 code standards. We also add several plugins to Flake8 to enforce additional +standards for GeoIPS code. Plugins used include: + +- `flake8-docstrings `_ is used to enforce + the numpy docstring standard. +- `flake8-rst-docstrings `_ is + used to ensure that docstrings are valid reStructuredText. +- `flake8-rst ` runs flake8 on code + snippets in reStructuredText files to ensure proper formatting in + documentation. + +We modify the default behavior of flake8 slightly to make it work well with Black, +ignore specific errors, and configure plugins. GeoIPS specific settings for +flake8 include the following: + +```toml +[flake8] +max-line-length=88 +count=True +ignore=E203,W503,E712 +extend-exclude=_version.py,lib,*_docs,geoips_dev_utils +docstring-convention=numpy +rst-roles=class,func,ref +rst-directives=envvar,exception +rst-substitutions=version +statistics=True +per-file-ignores = + /*/interfaces/__init__.py:F401 +``` + +Github Conventions +------------------ + +Pull Request Workflow +^^^^^^^^^^^^^^^^^^^^^ + +`https://nrlmmd-geoips.github.io/geoips/devguide/git_workflow.html#geoips-github-pull-request-workflow `__ + +`https://nrlmmd-geoips.github.io/geoips/devguide/git_workflow.html#geoips-merge-pr-and-close-issue-workflow `__ + +Issue Workflow +^^^^^^^^^^^^^^ + +`https://nrlmmd-geoips.github.io/geoips/devguide/git_workflow.html#geoips-github-issue-creation-workflow `__ + +Other Conventions +----------------- diff --git a/docs/source/_templates/index_PKG.html b/docs/source/_templates/index_PKG.html index 6867868e7..ac8f6988c 100644 --- a/docs/source/_templates/index_PKG.html +++ b/docs/source/_templates/index_PKG.html @@ -11,7 +11,9 @@ **Date**: |today| **Version**: |release| -**Download PDF documentation**: :download:`GeoIPS_PKGNAME.pdf` +.. + Temporarily comment out PDF documentation download. PDF build fails. + **Download PDF documentation**: :download:`GeoIPS_PKGNAME.pdf` **Previous versions**: Documentation of previous PKGNAME versions are available at `github.com/NRLMMD-GEOIPS `__. @@ -19,23 +21,16 @@ **Useful links**: `Source Repository `__ | `GeoIPS License `__ | -`NRLMMD `__ | +`NRLMMD `__ | -:mod:`PKGNAME` is a free software program, United States Government NRLMMD licensed. +:mod:`PKGNAME` is a free software program, using the NRL Open License Agreement. :: Distribution Statement A. Approved for public release. Distribution is unlimited. - Author: - Naval Research Laboratory, Marine Meteorology Division - - This program is free software: you can redistribute it and/or modify it under - the terms of the NRLMMD License included with this program. This program is - distributed WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the included license - for more details. If you did not receive the license, for more information see: - https://github.com/U-S-NRL-Marine-Meteorology-Division/ + This source code is protected under the license referenced at + https://github.com/NRLMMD-GEOIPS. .. automodule:: PKGNAME :noindex: @@ -122,9 +117,9 @@ :titlesonly: {% endif %} {% if not single_doc %} - introduction/index + INTROIDX STARTERIDX - userguide/index + USERGUIDEIDX DEVIDX DEPLOYGUIDEIDX OPGUIDEIDX diff --git a/docs/source/devguide/git_workflow.rst b/docs/source/devguide/git_workflow.rst index 2af6ee523..10ef4c72e 100644 --- a/docs/source/devguide/git_workflow.rst +++ b/docs/source/devguide/git_workflow.rst @@ -57,7 +57,7 @@ FROM WEB: MEMBERS: Create Branch from Existing Issue ==================================================== **NOTE: Those who are NOT members of the GeoIPS organization will fork,** -**not branch. Skip to NON MEMBERS section** +**not branch. If you are unable to branch, skip to NON MEMBERS section** * Navigate to Issue you would like to resolve * Click on Development->Create Branch @@ -68,15 +68,15 @@ FROM WEB: MEMBERS: Create Branch from Existing Issue * NOTE you can create branches on repositories outside the repository the Issue resides in - * **Change branch source** optional (defaults to "main") + * **Change branch source** optional (defaults to default branch) * Select **"Checkout locally"** * Click **"Create branch"** * Copy and paste the resulting "git fetch" and "git checkout" commands FROM WEB: NON-MEMBERS: Create fork of repo ========================================== -* **NOTE: NRLMMD-GEOIPS members will branch following steps above, - **skip this section if you are a member** +* **NOTE: GeoIPS organization members will branch following steps above, + **skip this section if you are a member and are able to branch** * Navigate to desired repository * Click drop down next to "Fork" * Click "+ Create a new fork" @@ -88,22 +88,22 @@ FROM WEB: NON-MEMBERS: Create fork of repo GeoIPS command line workflow **************************** -FROM COMMAND LINE: Switch to new branch, Make changes as usual -============================================================== +FROM COMMAND LINE: Switch to new branch/fork, Make changes as usual +=================================================================== * Navigate to repository of your choice * Issue only needs to be created on a single repository - * You can create branches and make changes on any number of repos, - as appropriate. + * You can create branches/forks and make changes on any number of + repos, as appropriate. * Related changes on different repositories will all be linked to the same Issue. -* Switch to new branch, and make changes as appropriate +* Switch to new branch/fork, and make changes as appropriate * *Ensure you copy and paste git fetch and git checkout commands* - *when creating branch above* - * Switch to new branch: Paste git fetch / git checkout commands + *when creating branch/fork above* + * Switch to new branch/fork: Paste git fetch / git checkout commands specified when creating branch from Issue * git fetch origin @@ -113,8 +113,8 @@ FROM COMMAND LINE: Switch to new branch, Make changes as usual * Use enforced commit message format for all commits - * Please follow - `Commit Message Template `_ + * Please follow `Commit Message Template + `_ * Summary line <= 120 characters * Blank line (if commit message is more than one line) * OPTIONAL: additional details @@ -124,7 +124,8 @@ FROM COMMAND LINE: Switch to new branch, Make changes as usual * Before pushing your final changes to GitHub and creating a pull request, you MUST update the release notes with your current changes. - * Instructions found in `CHANGELOG.rst `_ + * Instructions found in `CHANGELOG.rst + `_ * If release notes are not updated properly, pull request will not be approved. * Create test scripts and associated outputs for any new functionality @@ -147,8 +148,8 @@ Push changes to github GeoIPS GitHub Pull Request workflow *********************************** -FROM WEB: Create pull request from new ticket branch to "dev" branch -==================================================================== +FROM WEB: Create pull request from new ticket branch to default branch +====================================================================== Follow these instructions for each repo that requires changes for a given Issue. diff --git a/docs/source/introduction/index.rst b/docs/source/introduction/index.rst index e019cb71c..ba6fa4ee5 100644 --- a/docs/source/introduction/index.rst +++ b/docs/source/introduction/index.rst @@ -6,8 +6,6 @@ Introduction ************ -:ref:`requirement_spec` - .. toctree:: :maxdepth: 2 diff --git a/docs/source/new-docs/contribute/coding_standards.rst b/docs/source/new-docs/contribute/coding_standards.rst index 4fd997b7c..98016ac22 100644 --- a/docs/source/new-docs/contribute/coding_standards.rst +++ b/docs/source/new-docs/contribute/coding_standards.rst @@ -29,6 +29,300 @@ right in the case of a conflict with this document. External Style Standards ------------------------ +Numpy Docstrings +^^^^^^^^^^^^^^^^ + +Full information here: `Docstring Standard `_ + +Common mishaps: +""""""""""""""" + +The first docstring line should be a "one-line summary that **does not use variable +names or the function name**" + +Classes +""""""" + +.. dropdown:: Example + + .. code-block:: python + + class Calculator: + """A simple calculator class for performing basic arithmetic operations. + + This class provides methods to add, subtract, multiply, and divide + two numeric values. It also includes a `safe_divide` method to handle + division by zero more gracefully. + + .. deprecated:: 1.1.0 + The `divide` method will be removed in version 2.0.0. + Use `safe_divide` instead, which handles division by zero. + + Extended Summary + ---------------- + The `Calculator` class is designed for basic arithmetic operations + such as addition, subtraction, multiplication, and division. It is + useful for simple computational tasks and educational purposes. + + The `safe_divide` method is a safer version of `divide` that returns + `inf` when dividing by zero, rather than raising an error. + + Parameters + ---------- + log_operations : bool, optional + If True, all operations will be logged to a file (default is False). + + Attributes + ---------- + log_operations : bool + Indicates whether operations will be logged to a file. + + Other Parameters + ---------------- + output_format : str, optional + Format for the output of arithmetic operations. It must be one of + `{'decimal', 'fraction'}`. Default is 'decimal'. + + Raises + ------ + ZeroDivisionError + If `divide` is used with the second operand as zero. + + See Also + -------- + numpy.add : Element-wise addition for numpy arrays. + numpy.subtract : Element-wise subtraction for numpy arrays. + numpy.multiply : Element-wise multiplication for numpy arrays. + + Notes + ----- + This class is intended for scalar arithmetic operations. If you are + working with arrays, you should consider using `numpy` for vectorized + operations, which will be more efficient. + + References + ---------- + .. [1] Python Documentation: https://docs.python.org/3/library/operator.html + .. [2] NumPy Documentation: https://numpy.org/doc/stable/reference/routines.math.html + + Examples + -------- + Create a `Calculator` object and perform arithmetic operations: + + >>> calc = Calculator() + >>> calc.add(10, 5) + 15 + >>> calc.subtract(10, 5) + 5 + >>> calc.multiply(10, 5) + 50 + >>> calc.divide(10, 5) + 2.0 + >>> calc.safe_divide(10, 0) + inf + + """ + +Sections: + +#. `Short summary `_ +#. `Deprecation warning (optional) `_ +#. `Extended Summary `_ +#. `Parameters (optional) `_ +#. `Other Parameters (optional) `_ +#. `Raises (optional) `_ +#. `See Also (optional) `_ +#. `Notes (optional) `_ +#. `References (optional) `_ +#. `Examples `_ + +Methods +""""""" + +.. dropdown:: Example + + .. code-block:: python + + def matrix_multiply(a, b, out=None): + """Multiply two matrices. + + Computes the matrix product of two arrays `a` and `b`. If an output array + is provided, the result is stored in `out`. This function follows the + standard rules for matrix multiplication in linear algebra. + + .. deprecated:: 1.5.0 + This function will be removed in NumPy 2.0.0. + Use `numpy.matmul` or `numpy.dot` instead. + + Parameters + ---------- + a : array_like + The first matrix to be multiplied. + b : array_like + The second matrix to be multiplied. + out : ndarray, optional + If provided, the result will be stored in this array. It must have + the correct shape to store the result. + + Other Parameters + ---------------- + dtype : data-type, optional + If specified, forces the operation to cast the inputs to the given + type before performing the operation. + + Returns + ------- + output : ndarray + The matrix product of `a` and `b`. If `out` is provided, this array + is returned. + + Raises + ------ + ValueError + If the shapes of `a` and `b` are not aligned for matrix multiplication. + + See Also + -------- + numpy.matmul : Matrix product of two arrays. + numpy.dot : Dot product of two arrays. + numpy.einsum : Einstein summation convention. + + Notes + ----- + This function implements the matrix product as described in linear algebra. + It is different from element-wise multiplication of arrays. + + If either of the inputs is a scalar, it will be broadcast according to + standard broadcasting rules. + + References + ---------- + .. [1] Strang, G., "Introduction to Linear Algebra, 5th Edition," Wellesley-Cambridge Press, 2016. + + Examples + -------- + Multiply two 2x2 matrices: + + >>> import numpy as np + >>> a = np.array([[1, 2], [3, 4]]) + >>> b = np.array([[5, 6], [7, 8]]) + >>> matrix_multiply(a, b) + array([[19, 22], + [43, 50]]) + + Store result in a pre-allocated output array: + + >>> out = np.empty((2, 2)) + >>> matrix_multiply(a, b, out=out) + array([[19, 22], + [43, 50]]) + >>> out + array([[19., 22.], + [43., 50.]]) + + """ + import numpy as np + + a = np.asarray(a) + b = np.asarray(b) + + if out is None: + return np.dot(a, b) + else: + np.dot(a, b, out=out) + return out + + +Sections: + +#. `Short summary `_ +#. `Deprecation warning (optional) `_ +#. `Extended Summary `_ +#. `Parameters (optional) `_ +#. `Other Parameters (optional) `_ +#. `Raises (optional) `_ +#. `See Also (optional) `_ +#. `Notes (optional) `_ +#. `References (optional) `_ +#. `Examples `_ + +More sections necessary for generators/etc. please see the original numpy standards 😄 + +Modules +""""""" + +.. dropdown:: Example + + .. code-block:: python + + """A simple mathematics module for common operations. + + This module provides basic mathematical operations such as addition, subtraction, + multiplication, and division. It is designed to serve as a utility for quick calculations + without external dependencies. + + Extended Summary + ---------------- + The `mymathlib` module is created for educational purposes and provides a minimalistic + implementation of basic arithmetic operations. Each function performs a specific mathematical + task and can handle a wide range of input types, including integers and floats. This module + is intentionally simple to demonstrate NumPy-style documentation and function listings. + + Routine Listings + ---------------- + add(a, b) + Return the sum of `a` and `b`. + + subtract(a, b) + Return the result of `a` minus `b`. + + multiply(a, b) + Return the product of `a` and `b`. + + divide(a, b) + Return the result of `a` divided by `b`. + + See Also + -------- + numpy.add : Adds two arrays element-wise. + numpy.subtract : Subtracts one array from another element-wise. + numpy.multiply : Multiplies two arrays element-wise. + numpy.divide : Divides two arrays element-wise. + + Notes + ----- + This module does not handle complex numbers or provide error handling for division + by zero. It assumes valid inputs (integers or floats) for all functions. + + References + ---------- + .. [1] NumPy documentation, https://numpy.org/doc/stable/reference/routines.math.html + .. [2] Python official documentation, https://docs.python.org/3/library/math.html + + Examples + -------- + >>> from mymathlib import add, subtract, multiply, divide + >>> add(2, 3) + 5 + >>> subtract(10, 5) + 5 + >>> multiply(4, 3) + 12 + >>> divide(9, 3) + 3.0 + """ + + +Sections: + +#. `Short summary `_ +#. `Extended summary (optional) `_ +#. `Routine listings (optional) `_ +#. `See also (optional) `_ +#. `Notes (optional) `_ +#. `References (optional) `_ +#. `Examples `_ + `PEP 8 `__ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/new-docs/getting-started/best-practices/best_practices.rst b/docs/source/new-docs/getting-started/best-practices/best_practices.rst new file mode 100644 index 000000000..fc064a701 --- /dev/null +++ b/docs/source/new-docs/getting-started/best-practices/best_practices.rst @@ -0,0 +1,210 @@ +:orphan: + +.. dropdown:: Distribution Statement + + | # # # This source code is protected under the license referenced at + | # # # https://github.com/NRLMMD-GEOIPS. + +.. _best_practices: + +GeoIPS Generalized Best Practices +================================= + +There are many ways one can incorporate GeoIPS into their research or operational flow. +While the developers do not want to be prescriptivist, we do recommend a series +of best practices to ensure that your plugin development/deployment experience is as smooth as possible. +First, we will propose some generalized best practices to consider before developing a plugin, +followed by best practices to follow during development, during initial release/deployment, +and finally at the planned or unplanned EOL of your plugin. Future iterations of this document +will include best practices for operational and research use of GeoIPS. + +Best Practices at Conception of your Plugin +------------------------------------------- + +In general, try to leverage existing GeoIPS capabilities before developing your own plugins! + +* One of the core functions of GeoIPS is to eliminate redundant development time. Leverage that! +* If a reader exists, but is missing a core functionality required for your development, consider the + following options: + + * Reach out to the developer of the reader and request they add the feature(s) you are missing. Odds + are good that if you need it, someone else probably will too! + * Pull the plugin containing the reader and add the functionality yourself. Then, open a PR on the repo + containing the plugin to merge your changes. + * If all else fails and the reader is no longer actively maintained, create a local copy for your plugin + and provide proper attribution somewhere in your repository (provided you're going to open source your + plugin, of course.) + +Try to make an effort to attend the weekly open-source meetings routinely and become an active +participant in the discussions! As a community-driven effort, GeoIPS thrives as more people +join in! + +* This will ensure you're up to date on the latest developments and are aware of what's coming down + the pipeline. See the first bullet point for why that's important! +* You can also make requests of folks that are developing plugins and products to consider sources + you are interested in working with. + + * This means you may not have to do as much development work on any plugins you develop + * This also works both ways-- you might be able to save a member of the community some effort, + as well as drive some collaboration! +* You'll be able to provide guidance on the direction GeoIPS is heading, ensuring that it is robust + and has broad spectrum utility. + + * One prominent example includes the efforts of Tim Olander and Tony Wimmers's provision of + feedback during development of the plugin system. Their feedback helped shape our overall + implementation of plugins, and included one of the first demonstrations of the system via ARCHER. + * Another example includes the development efforts at CIRA by Evan Rose and Jeremy Solbrig, who, + while working closely with the community, have helped shape the way users interact with GeoIPS + from the lowest levels, to the highest levels. +* Lastly, you can share your success stories and drive interest in the plugins you are developing. + +Best Practices During Development of your Plugin +------------------------------------------------ + +When you develop a plugin, it is best to BEGIN by following the coding and formatting standards +recommended by the GeoIPS developers to facilitate easy sharing and integration into the GeoIPS +infrastructure. + +* Make sure that you're developing in your own repository and not on the GeoIPS main branch. +* Ensure your code is properly commented and legible for fellow developers. You never know how many + emails you'll be able to prevent by explaining your logic in advance! +* Make sure you follow the standards and practices recommended for providing informative runtime + messages, and that they are logged at the correct levels. + + * Be sure to include a healthy mix of INFO, INTERACTIVE, DEBUG, WARNING and ERROR-level messages! +* Consider running an automated system to monitor your code for formatting and potential security + risks (e.g., SQL injection.) + + * check_code.sh is a GeoIPS built-in method for monitoring these! + * This is especially crucial for your code to enter operations on controllsed systems e.g., FNMOC, + NAVO. + +Packages change frequently! Make sure you're always on the latest working branch of GeoIPS and +any plugins you are using before sharing your code! + +* Sometimes interfaces are deprecated between versions-- it's easier for everyone if you make sure + everything works on the latest version. + + * An upcoming GeoIPS 2.0 will retain backward functionality for most plugins, however we reserve + the right to revoke that capability at any time. +* To that end, stay abreast of upcoming deprecations within non-GeoIPS packages used by your plugin. + You never know when NumPy will finally make good on all those warnings you've been ignoring! + +We strongly discourage developing new, core infrastructure for GeoIPS, including new interfaces +and subsystems. + +* These kinds of developments have a strong chance of creating conflicts later on down the line that + may be difficult to resolve against the main GeoIPS repository. +* If you identify a need for new, core functionality raise your concern/request in the weekly meetings. + At best we may have an alternative option to recommend, and at worst you may be conscripted into + developing the feature! + +If your GeoIPS plugin contains ANY proprietary IP or other sensitive data, it is CRUCIALLY IMPORTANT +that you do not make any changes to the codebases of the main GeoIPS repository or any plugins you +are using! + +* In general, we will reject most commits made the main GeoIPS repository +* If that chance exists that your repo will enter open source, ensure that you are not committing + anything private or dangerous. + + * Check for things like private keys in the commits, small pieces of information that when + combined could lead to spillage, nasty merge comments, etc. + +Ensure that your code is performant and reasonably Pythonic. Nobody wants their code to be the +bottleneck in someone else's workflow! + +* If your plugin includes C or Fortran-wrapped algorithms that's also okay! We support interfaces + that use f2py, etc. You might consider converting to native Python later on down the line, but + that's never going to be mandatory. + +Include test scripts and archive the mininmum number of files required to test your plugin! + +* Doing so provides your users a quick way to understand how your plugin is invoked. + + * We recommend passing explicit file inputs to the procflow rather than a directory to ensure + exact testing. +* The ability of developers to validate their installation of your plugin is crucial. + + * To that end, consider including the final output images to be used in the + +Lastly, share your successes and struggles at the weekly open source meeting! + +* If you're running into problems or have questions, this is one of the best ways to resolve them. + + * You can also join the GeoIPS slack to ask questions 24/7 +* People may have good suggestions or more performant ways to accomplish your goals. +* People will love to see the pretty pictures you make. + +Best Practices Before and After Deploying your Plugin +----------------------------------------------------- + +While we can't control how your administrate your own repositories, or how you run +GeoIPS, we can make some suggestions! + +When you commit to an internal or public-facing repository, make sure ownership and permissions are +controlled appropriately. + +* Try to not let people freely merge code into your repository as it creates a security risk for + you and anyone using your plugin. +* Establish review requirements early on, and ensure that commits and PRs are fully tested before + being merged. Optionally (but ideally,) you'd test these on a fresh GeoIPS installation to catch + any dependencies that may be missing. + + * This is a good time to again ensure you're developing on the most recent versions of the GeoIPS + and plugins. +* Include documentation for your plugin! + + * You can use this repository's documentation as a standard. + +The infrastructure for automated processing and scheduling of GeoIPS runs on your system is +outside the scope this documentation. + +* We don't want to risk pigeonholing anyone or exposing our own internal systems. +* That said, in a future release we may make some basic repositories with simple database + and automation packages available. +* In general, though, all you need is way to identify when new files are available and a way + to automate the passing of those files to GeoIPS. There are myriad solutions from this ranging + from simple (cron) to advanced (Cylc) + +Let people know when your plugin is ready for release! + +* Share the good news at the open source meetings! Give talks at conferences! Advertise as + best as you can! + + * This is good for GeoIPS and good for you. You may wind up roping in new talent or find + transition avenues you weren't aware of. + +Finally, be reasonably available to support your plugin for end users. + +* Try to be an active participant on your repository's issues board at minimum. +* Answer questions that people have and address issues they may be coming across. +* Communicate your planned update schedule (if any) and try to stick to it. + + * If you plan to deprecate features of your plugin or bring it to end-of-life (EOL,) + be sure to announce that well in advance! + +Best Practices for Deprecation/EOL of your Plugin +------------------------------------------------- + +When your plugin crosses the rainbow bridge, due to loss of funding, personnel, or interest +consider the following: + +* Announce the upcoming deprecation/EOL at the GeoIPS weekly open source meeting. +* Include in your plugin repository a notice of upcoming EOL. +* Establish a chain of ownership/maintenance if possible. + + * Have in place a method for another developer to take control of the repository + for ongoing updates + * Remove or automatically redirect the repository to one where active development + of the plugin is ongoing + * Failing the above, place a prominent warning banning that indicates active + development of the plugin has ceased. + +GeoIPS Best Practices for Operations +====================================== +[COMING SOON] + +GeoIPS Best Practices for Research +====================================== +[COMING SOON] diff --git a/docs/source/new-docs/getting-started/installing/test-data.rst b/docs/source/new-docs/getting-started/installing/test-data.rst new file mode 100644 index 000000000..c56788f70 --- /dev/null +++ b/docs/source/new-docs/getting-started/installing/test-data.rst @@ -0,0 +1,8 @@ +:orphan: + +.. dropdown:: Distribution Statement + + | # # # This source code is protected under the license referenced at + | # # # https://github.com/NRLMMD-GEOIPS. + + diff --git a/docs/source/releases/1.14.0/1.14.0.yaml b/docs/source/releases/1.14.0/1.14.0.yaml new file mode 100644 index 000000000..3035eb301 --- /dev/null +++ b/docs/source/releases/1.14.0/1.14.0.yaml @@ -0,0 +1,79 @@ +Installation: +- title: 'Remove PATH and PYTHON path from setup configs' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + This was causing problems with using the wrong python/pip during processing. + Do not explicitly set PATH or PYTHONPATH. + files: + added: + - '' + deleted: + - '' + modified: + - 'setup/config_geoips' + - 'setup/geoips_conda_init_setup' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' +GitHub Actions: +- title: 'Turn off pdf doc build from doc-test' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + We still need to fix the pdf doc build, but turning it off temporarily until + we determine what the issue is. + files: + added: + - '' + deleted: + - '' + modified: + - '.github/workflows/doc-test.yaml' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' +- title: 'Turn off RST release note check from version release branches' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + Since version releases often have corrections to RST release notes, do not fail + on RST release note check within version release branches. + files: + added: + - '' + deleted: + - '' + modified: + - '.github/workflows/release-note-check.yaml' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' +- title: 'Move 1.14.0 and 1.14.1 rst files into latest/*.yaml and upcoming/*.yaml' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + No longer deleting YAML files or checking in rst files - store + all release notes as YAML in perpetuity, and auto-generate the + rst with brassy every time. This will make rolling back releases + MUCH easier. + files: + added: + - '' + deleted: + - 'docs/source/releases/v1_14_0.rst' + - 'docs/source/releases/v1_14_1.rst' + modified: + - 'docs/source/releases/latest/v1_14_0.yaml' + - 'docs/source/releases/upcoming/v1_14_1.yaml' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' \ No newline at end of file diff --git a/docs/source/releases/1.14.0/1.14.0_environment_files.yaml b/docs/source/releases/1.14.0/1.14.0_environment_files.yaml new file mode 100644 index 000000000..8df16efc9 --- /dev/null +++ b/docs/source/releases/1.14.0/1.14.0_environment_files.yaml @@ -0,0 +1,25 @@ +Release Process: + - title: "Add pip and mamba 1.14.0 environment files" + description: | + *From GEOIPS#562: 2024-09-17, 1.14.0 updates* + + Package: geoips_full + Total run time: 2558 seconds + Number data types run: 56 + Number data types failed: 0 + Mon Sep 16 22:06:05 UTC 2024 + + Package: geoips_base + Total run time: 156 seconds + Number data types run: 3 + Number data types failed: 0 + Tue Sep 17 01:45:03 UTC 2024 + files: + added: + - environments/mamba_base_package_list_1.14.0_20240915.yml + - environments/mamba_full_package_list_1.14.0_20240916.yml + - environments/pip_base_requirements_1.14.0_20240915.txt + - environments/pip_full_requirements_1.14.0_20240916.txt + related-issue: + number: 562 + repo_url: "GEOIPS/geoips" diff --git a/docs/source/releases/1.14.0/build-all-rst-release-notes-every-time.yaml b/docs/source/releases/1.14.0/build-all-rst-release-notes-every-time.yaml new file mode 100644 index 000000000..bb54f6ee5 --- /dev/null +++ b/docs/source/releases/1.14.0/build-all-rst-release-notes-every-time.yaml @@ -0,0 +1,38 @@ +Documentation: +- title: 'Build RST release notes from YAML files every time' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + Rather than generating and checking in RST files with + each release, save all the original YAML files and produce + RST every time from within build_docs.sh. + + Store the release notes in X.Y.Z directories, producing + X.Y.Z.rst during the build_docs process. Update the index.rst + generating code to support either vX_Y_Z.rst or X.Y.Z.rst + filename format. + + This allows simply moving the 'latest' release note directory + to X.Y.Z during the release process, and maintaining a record + of all the original release notes in perpetuity, facilitating + rapid release roll backs or evaluation of previous release notes. + + Note we also now ignore an 'upcoming' release note directory, + as that is a temporary location to store the release note for + the upcoming release during the version release process for + the current version ('latest'). At the end of the release + process, the 'upcoming' directory will be moved to 'latest' + to prepare for the next release. + files: + added: + - '' + deleted: + - '' + modified: + - 'docs/build_docs.sh' + - 'docs/update_release_note_index.py' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' diff --git a/docs/source/releases/1.14.0/fix-pdf-doc-build.yaml b/docs/source/releases/1.14.0/fix-pdf-doc-build.yaml new file mode 100644 index 000000000..e8ce13b92 --- /dev/null +++ b/docs/source/releases/1.14.0/fix-pdf-doc-build.yaml @@ -0,0 +1,25 @@ +documentation: +- title: 'Resolve pdf doc build issues' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + PDF doc builds often fail when html succeeds - this MUST be tested regularly + on PR merge, to ensure we keep up with pdf builds as well as html. Much easier to + resolve issues as they crop up rather than much later when we realize it's failing. + + Temporarily turn off all pdf builds, and comment out the pdf download from the + documentation index template. + files: + added: + - '' + deleted: + - '' + modified: + - '.github/workflows/doc-test.yaml' + - '.github/workflows/reusable-deploy-docs.yaml' + - 'docs/source/_templates/index_PKG.html' + moved: + - '' + related-issue: + number: 562 + repo_url: '' diff --git a/docs/source/releases/1.14.0/make_all_doc_sections_optional.yaml b/docs/source/releases/1.14.0/make_all_doc_sections_optional.yaml new file mode 100644 index 000000000..cdbc05d6b --- /dev/null +++ b/docs/source/releases/1.14.0/make_all_doc_sections_optional.yaml @@ -0,0 +1,21 @@ +documentation: +- title: 'Make all documentation sections optional' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + Previously required introduction and user guide - allow building documentation + with only release notes and API. + files: + added: + - '' + deleted: + - '' + modified: + - 'docs/build_docs.sh' + - 'docs/source/_templates/index_PKG.html' + moved: + - '' + related-issue: + number: 562 + repo_url: '' + diff --git a/docs/source/releases/1.14.0/move-reusable-workflows-and-sync-to-geoips_ci.yaml b/docs/source/releases/1.14.0/move-reusable-workflows-and-sync-to-geoips_ci.yaml new file mode 100644 index 000000000..207e2af97 --- /dev/null +++ b/docs/source/releases/1.14.0/move-reusable-workflows-and-sync-to-geoips_ci.yaml @@ -0,0 +1,57 @@ +GitHub Actions: +- title: 'Move reusable workflows and sync files to geoips_ci' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + Create a new repository, geoips_ci, that will manage all GeoIPS Ecosystem + reusable workflows, as well as files to sync to all GeoIPS plugin + repositories. + + Simplify brassy-notes and tag-and-release to filter on pr merged immediately + (rather than trying to pass pr merged flag into reusable workflow). These should + never run on a closed but not merged PR. + files: + added: + - .github/workflows/brassy-notes.yaml + - .github/workflows/deploy-docs.yaml + - .github/workflows/package-and-publish.yaml + - .github/workflows/tag-and-release.yaml + deleted: + - .github/sync/.flake8 + - .github/sync/.github/release.yml + - .github/sync/.github/workflows/codeql.yml + - .github/sync/.gitignore + - .github/sync/CHANGELOG.rst + - .github/workflows/reusable-brassy-notes.yaml + - .github/workflows/reusable-check-release-branch.yaml + - .github/workflows/reusable-deploy-docs.yaml + - .github/workflows/reusable-get-versions.yaml + - .github/workflows/reusable-package-and-publish.yaml + - .github/workflows/reusable-print-variables.yaml + - .github/workflows/reusable-tag-and-release.yaml + modified: + - '' + moved: + - '' + related-issue: + number: 562 + repo_url: '' +- title: 'Add standard synced files to geoips repo' + description: | + *From issue GEOIPS#562: 2024-08-15, version update* + + These files auto-copied from geoips_ci. + files: + added: + - .flake8 + - pytest.ini + - bandit.yml + deleted: + - '' + modified: + - 'CHANGELOG.rst' + moved: + - '' + related-issue: + number: 562 + repo_url: '' diff --git a/docs/source/releases/1.14.1/1.14.1-release-note.yaml b/docs/source/releases/1.14.1/1.14.1-release-note.yaml new file mode 100644 index 000000000..189ffba78 --- /dev/null +++ b/docs/source/releases/1.14.1/1.14.1-release-note.yaml @@ -0,0 +1,26 @@ +documentation: + - title: Add release note for version 1.14.1 + description: | + *From GEOIPS#562: 2024-09-24, 1.14.1 updates* + Release note on release branch to force tag/release, package/publish, + and deploy docs. + files: + added: + - docs/source/releases/latest/1.14.1-release-note.yaml + related-issue: + number: 562 + repo_url: GEOIPS +documentation: + - title: Update upcoming and tagged versions + description: | + *From GEOIPS#562: 2024-09-25, 1.14.1 updates* + + * Tagged version: 1.14.1 + * Upcoming version: 1.14.2 + files: + modified: + - .github/versions/tagged_version + - .github/versions/upcoming_version + related-issue: + number: 562 + repo_url: GEOIPS diff --git a/docs/source/releases/1.14.1/1.14.1.yaml b/docs/source/releases/1.14.1/1.14.1.yaml new file mode 100644 index 000000000..67daaaae7 --- /dev/null +++ b/docs/source/releases/1.14.1/1.14.1.yaml @@ -0,0 +1,19 @@ +Release Process: +- title: 'Update tagged_version and upcoming_version' + description: | + *From GEOIPS#562: 2024-08-13, 1.13.2 updates* + + Currently tagged version 1.14.0, upcoming version 1.14.1. + files: + added: + - '' + deleted: + - '' + modified: + - '.github/versions/tagged_version' + - '.github/versions/upcoming_version' + moved: + - '' + related-issue: + number: 562 + repo_url: 'GEOIPS/geoips' diff --git a/docs/source/releases/index.rst b/docs/source/releases/index.rst deleted file mode 100644 index e7d51daf0..000000000 --- a/docs/source/releases/index.rst +++ /dev/null @@ -1,232 +0,0 @@ -.. dropdown:: Distribution Statement - - | # # # This source code is protected under the license referenced at - | # # # https://github.com/NRLMMD-GEOIPS. - -.. _release_notes: - -Release Notes -************* - -Version 1.14 ------------- - -.. toctree:: - :maxdepth: 1 - - v1_14_0a0 - -Version 1.13 ------------- - -.. toctree:: - :maxdepth: 1 - - v1_13_1 - v1_13_0 - v1_13_0a0 - -Version 1.12 ------------- - -Added documentation and enhancements for coding standards, fixed bugs in polar data, log functions, and Dockerfile; -Introduced documentation for PEP8, linters, and plugin registries; Refactored optional imports, plugin registries. Fixed -issues with create_plugin_registries, ATMS platform names, and pytest dependencies. - -.. toctree:: - :maxdepth: 1 - - v1_12_4 - v1_12_3 - v1_12_2 - v1_12_2a0 - v1_12_1 - v1_12_0 - -Version 1.11 ------------- - -Removed flake8_docstring_only flag from test_all.sh and full_test.sh across the repository. Fixed issues in integration -tests and ensured full flake8 tests applied. - -.. toctree:: - :maxdepth: 1 - - v1_11_7 - v1_11_7a0 - v1_11_6 - v1_11_5 - v1_11_5a0 - v1_11_3 - v1_11_3a0 - v1_11_2 - v1_11_1 - v1_11_1a0 - v1_11_0 - -Version 1.10 ------------- - -Updated interface names throughout the code base, moved YAML sector files to new directories within the geoips/plugins -structure, added support for both the old and new dynamic sector YAML formatting, and renamed plugins to enhance -clarity. Introduced a try/except block in sector_utils/tc_tracks.py to support both old and new dynamic sector YAML -formats. Consolidated both dynamic and static sectors under the unified directory structure in -geoips/plugins/yaml/sectors, and renamed various modules to reflect their functionalities more accurately. - -.. toctree:: - :maxdepth: 1 - - v1_10_3 - v1_10_2 - v1_10_1 - v1_10_0 - v1_10_0a13 - v1_10_0a12 - v1_10_0a11 - v1_10_0a10 - v1_10_0a9 - v1_10_0a8 - v1_10_0a7 - v1_10_0a6 - v1_10_0a5 - v1_10_0a4 - v1_10_0a3 - v1_10_0a2 - v1_10_0a1 - v1_10_0a0 - -Version 1.9 ------------ - -Removed GEOIPS BASEDIR references and introduced GEOIPS_TESTDATA_DIR and GEOIPS_DEPENDENCIES_DIR in -filenames/base_paths.py, updated and implemented sphinx-apidoc for Sphinx documentation builds, finalized RST -documentation outline to align with SharePoint documentation, replaced setup.py with pyproject.toml for project setup, -added "cbar_tick_labels" and "cbar_spacing" fields in Rain colormap mpl_color_info, allowed processing when no tropical -cyclones are present in the database, installed geoips[doc, test, lint] dependencies in setup.sh, and configured code -style to ignore flake8 W503 due to conflicts with black. - -.. toctree:: - :maxdepth: 1 - - v1_9_2 - v1_9_1 - v1_9_0 - -Version 1.8 ------------ - -Renamed SSMI/S 89GHz products to 91GHz, updated F-16 SSMI/S scaling, resolved basename error, added geoips_clavrx to -test_full_install, and corrected minor sphinx build warnings. Fixed tests, ensured accurate product naming, and improved -installation and documentation processes. - -.. toctree:: - :maxdepth: 1 - - v1_8_1 - v1_8_0 - -Version 1.7 ------------ - -Updated matplotlib version dependency, extended SAR incident angle range, and converted CHANGELOG and documentation -formats to RST. Removed outdated sections, fixed minor documentation issues, and improved documentation organization and -building process. - -.. toctree:: - :maxdepth: 1 - - v1_7_0 - -Version 1.6 ------------ - -Added resource usage statistics, documentation updates, and code style enforcement checks. Fixed numpy builtin aliases, -product family typos, and output requirements. - -.. toctree:: - :maxdepth: 1 - - v1_6_3 - v1_6_1 - v1_6_0 - -Version 1.5 ------------ - -Updated the Python installation process, added alternate command line arguments functions, created unique storm -directory names for invests, and fixed bugs. Expanded test repository updates and improved metadata handling. - -.. toctree:: - :maxdepth: 1 - - v1_5_3 - v1_5_2 - v1_5_1 - v1_5_0 - -Version 1.4 ------------ - -Updated documentation and installation requirements, added new test scripts and functionality. Fixed real-time -processing and coverage checks. - -.. toctree:: - :maxdepth: 1 - - v1_4_8 - v1_4_7 - v1_4_6 - v1_4_5 - v1_4_4 - v1_4_3 - v1_4_2 - v1_4_1 - v1_4_0 - -Version 1.3 ------------ - -Replaced instances of 'atcf' with 'tc', removed support for GEOIPSFINAL and GEOIPSTEMP, and consolidated directories -into 'geoips_outdirs.' Added dev/title.py interface, updated template pull request ticket, allowed passing -'title_copyright,' and added new DATABASESUCCESS and DATABASEFAILURE string checks. - -.. toctree:: - :maxdepth: 1 - - v1_3_2 - v1_3_1 - v1_3_0 - -Version 1.2 ------------ - -Updated bdeck parser, SMOS text winds, test_interfaces.py, attribute names, call signatures; added ABI test scripts, -Software Requirements Specification. Replaced original_source_filename with original_source_filenames; simplified setup, -command line args checking, installation steps. - -.. toctree:: - :maxdepth: 1 - - v1_2_5 - v1_2_4 - v1_2_3 - v1_2_2 - v1_2_1 - v1_2_0 - -Version 1.1 ------------ - -Addition of new features like MODIS reader, stitched product output, sector overpass database, and Visible global -stitched. Refactoring for Python 3 operation, and resolving errors in MODIS processing and RGB MTIF color issue. There -are also breaking changes including modularized PMW algorithms, changed dictionary structures, and standardized platform -names. - -.. toctree:: - :maxdepth: 1 - - v1_1_17 - v1_1_16 - v1_1_15 - v1_1_14 - v1_1_13 diff --git a/docs/source/releases/latest/0-pre-existing-conditions.yaml b/docs/source/releases/latest/0-pre-existing-conditions.yaml deleted file mode 100644 index 6420024a7..000000000 --- a/docs/source/releases/latest/0-pre-existing-conditions.yaml +++ /dev/null @@ -1,40 +0,0 @@ -bug-fix: -- description: | - Correct typo causing product family xrdict_area_product_to_outlist failure. Product - families that did not have pre-defined output lists and expected data passed - unmodified from the readers to the output formatters were failing due to a missing - else statement ensuring the list of final_products was set even when the output file - list was not pre-defined. Add else statement to set final_products to curr_products - when output file list not defined. This resolved an error with product family - xrdict_area_product_to_outlist when it was requested without sectoring, but it - worked if sectoring was requested (followed a different route through the code for - each). - title: "Correct typo causing product family xrdict_area_product_to_outlist failure" - related-issue: "GEOIPS#666 - Final Bug Fixes Prior to Workshop" - files: - modified: ["geoips/plugins/modules/procflows/single_source.py"] - -- description: | - Fix failing test data CLI unit tests. `test_geoips_config_install.py` and - `test_log_setup.py` had 1 or more tests that were failing, which caused the CI to - fail as well. Fix these unit tests so we can actually use the CI to address problems - that are coming from new PRs. - title: "Fix failing test data CLI unit tests" - related-issue: "GEOIPS#666 - Final Bug Fixes Prior to Workshop" - files: - modified: ["tests/unit_tests/commandline/test_log_setup.py", "tests/unit_tests/commandline/cli_top_level_tester.py"] - deleted: ["tests/unit_tests/commandline/test_geoips_config_install.py"] - -documentation: -- description: "Add pinkrst and brassy to dependencies." - title: "Add pinkrst and brassy to dependencies" - related-issue: "GEOIPS#675 - Add brassy and pinkrst to geoips lint dependencies" - files: - modified: ["pyproject.toml"] - -- description: "Update LICENSE and DISTRIBUTION to latest approved version." - title: "Update LICENSE and DISTRIBUTION to latest approved version" - related-issue: "GEOIPS#666 - Final Bug Fixes Prior to Workshop" - files: - modified: ["LICENSE", "DISTRIBUTION"] - diff --git a/docs/source/releases/latest/495-change-geoips-get-to-geoips-describe.yaml b/docs/source/releases/latest/495-change-geoips-get-to-geoips-describe.yaml deleted file mode 100644 index a4efb3640..000000000 --- a/docs/source/releases/latest/495-change-geoips-get-to-geoips-describe.yaml +++ /dev/null @@ -1,34 +0,0 @@ -Refactoring Updates: -- description: | - *From GEOIPS#495: 2024-04-15, Change CLI Command 'geoips get' to 'geoips describe'* - - This update renames all references of 'geoips get' to 'geoips describe'. 'Describe' - is a better name for this command as it generally provides additional information - about a certain GeoIPS artifact, whereas 'get' could be a bit ambiguous as users - may think we are actually getting a certain plugin, interface, etc. Another reason - for this update is that all references to the CLI in the tutorial use - 'geoips describe', rather than 'geoips get'. This PR was only a nomenclature change - and did not impact any of the actual functionality of the CLI. - files: - added: - - geoips/commandline/geoips_describe.py - - tests/unit_tests/commandline/test_geoips_describe_family.py - - tests/unit_tests/commandline/test_geoips_describe_interface.py - - tests/unit_tests/commandline/test_geoips_describe_package.py - - tests/unit_tests/commandline/test_geoips_describe_plugin.py - deleted: - - geoips/commandline/geoips_get.py - - tests/unit_tests/commandline/test_geoips_get_family.py - - tests/unit_tests/commandline/test_geoips_get_interface.py - - tests/unit_tests/commandline/test_geoips_get_package.py - - tests/unit_tests/commandline/test_geoips_get_plugin.py - modified: - - docs/source/userguide/command_line.rst - - geoips/commandline/ancillary_info/alias_mapping.yaml - - geoips/commandline/ancillary_info/cmd_instructions.yaml - - geoips/commandline/commandline_interface.py - - geoips/commandline/geoips_command.py - related-issue: - number: 495 - repo_url: 'https://github.com/NRLMMD-GEOIPS/geoips/' - title: 'Change geoips get to geoips describe' diff --git a/docs/source/releases/latest/575-cli-class-factory.yaml b/docs/source/releases/latest/575-cli-class-factory.yaml deleted file mode 100644 index b2038476c..000000000 --- a/docs/source/releases/latest/575-cli-class-factory.yaml +++ /dev/null @@ -1,27 +0,0 @@ -Refactoring Updates: -- description: | - *From GEOIPS#576: 2024-05-11, CLI: Reduce number of command layers where possible and appropriate* - *From GEOIPS#573: 2024-05-11, Use class factories to generate subcommands for commands like ``geoips get family`` and ``geoips get interface``* - - While this PR doesn't fix all of the problems addressed in the issues above, it is our - initial attempt at creating a class factory for certain CLI commands to reduce the - verbage needed to execute those commands. Spefically in this branch, we've addressed the - command class ``GeoipsListSingleInterface``. We now use the aforementioned class as a - base class to build ``GeoipsListSingleInterface`` classes at runtime. - - By doing so, we can now run ``geoips list `` rather than - ``geoips list interface ``. This improves the readability of this - command and makes it much easier to type via the CLI. We expect in future PRs to address - similar commands, such as ``geoips get family `` and - ``geoips get plugin `` for the reasons mentioned - previously. - title: 'Refactored ``geoips list interface`` to use a class factory' - files: - modified: - - docs/source/userguide/command_line.rst - - geoips/commandline/ancillary_info/cmd_instructions.yaml - - geoips/commandline/geoips_command.py - - geoips/commandline/geoips_list.py - - tests/unit_tests/commandline/cli_top_level_tester.py - - tests/unit_tests/commandline/test_geoips_list_interface.py - diff --git a/docs/source/releases/latest/576-get-cmd-class-factories.yaml b/docs/source/releases/latest/576-get-cmd-class-factories.yaml deleted file mode 100644 index e16838639..000000000 --- a/docs/source/releases/latest/576-get-cmd-class-factories.yaml +++ /dev/null @@ -1,64 +0,0 @@ -Refactoring Updates: -- description: | - *From GEOIPS#576: 2024-05-11, CLI: Reduce number of command layers where possible and appropriate* - *From GEOIPS#573: 2024-05-11, Use class factories to generate subcommands for commands like ``geoips get family`` and ``geoips get interface``* - - This branch refactors commands which use the form of ``geoips get `` to - make use of class factories. Before, we had to run commands such as: - - * ``geoips get interface `` - * ``geoips get family `` - * ``geoips get plugin `` - - The aforementioned commands were overly verbose and needed some refactoring to reduce - the levels required to execute a certain command. In this branch, we refactored - ``GeoipsGetInterface`` to be a base command for ``get ``, - ``get family ``, and - ``get ``. This resulted in the removal of - ``GeoipsGetFamily`` and ``GeoipsGetPlugin`` and the refactoring of - ``GeoipsGetInterface`` to include the functionality of those other commands. We can use - class factories for each interface to execute these commands now. - - On top of this, we implemented aliases for a variety of CLI commands. For an exact - listing of aliases available for each command, see - ``geoips/commandline/ancillary_info/alias_mapping.yaml``, which includes aliases - supported for each command. Now, we can run a verbose command such as: - - * ``geoips list algorithms -p geoips`` - - and replace it with - - * ``geoips ls algs -p geoips`` - - This also reduces the verbosity of certain CLI commands. Note, the unit tests have been - modified to reflect these changes. - files: - modified: - - .gitignore - - pyproject.toml - - docs/source/_templates/conf_PKG.py - - docs/source/userguide/command_line.rst - - geoips/commandline/ancillary_info/cmd_instructions.yaml - - geoips/commandline/cmd_instructions.py - - geoips/commandline/commandline_interface.py - - geoips/commandline/geoips_command.py - - geoips/commandline/geoips_get.py - - tests/unit_tests/commandline/cli_top_level_tester.py - - tests/unit_tests/commandline/test_geoips_get_family.py - - tests/unit_tests/commandline/test_geoips_get_interface.py - - tests/unit_tests/commandline/test_geoips_get_package.py - - tests/unit_tests/commandline/test_geoips_get_plugin.py - - tests/unit_tests/commandline/test_geoips_list_interface.py - - tests/unit_tests/commandline/test_geoips_list_interfaces.py - - tests/unit_tests/commandline/test_geoips_list_packages.py - - tests/unit_tests/commandline/test_geoips_list_plugins.py - - tests/unit_tests/commandline/test_geoips_list_scripts.py - - tests/unit_tests/commandline/test_geoips_list_test_datasets.py - - tests/unit_tests/commandline/test_geoips_list_unit_tests.py - - tests/unit_tests/commandline/test_get_commandline_instructions.py - added: - - geoips/commandline/ancillary_info/alias_mapping.yaml - related-issue: - number: 573 - repo_url: 'https://github.com/NRLMMD-GEOIPS/geoips' - title: 'Refactored ``geoips get `` commands to use a class factory' diff --git a/docs/source/releases/latest/608-documentation-system-requirements-for-geoips.yaml b/docs/source/releases/latest/608-documentation-system-requirements-for-geoips.yaml deleted file mode 100644 index 03b4033a0..000000000 --- a/docs/source/releases/latest/608-documentation-system-requirements-for-geoips.yaml +++ /dev/null @@ -1,18 +0,0 @@ -documentation: -- description: | - Added system requirements to the documentation. This includes minimum and recommended - requirements for users and developers as well as language limiting the applicability - of the system requirements. - files: - added: - - '' - deleted: - - '' - modified: - - 'docs/source/new-docs/getting-started/system-requirements/index.rst' - moved: - - '' - related-issue: - number: 608 - repo_url: '' - title: 'Write System Requirements' diff --git a/docs/source/releases/latest/627-CLI-tree.yaml b/docs/source/releases/latest/627-CLI-tree.yaml deleted file mode 100644 index 6fcab8ab5..000000000 --- a/docs/source/releases/latest/627-CLI-tree.yaml +++ /dev/null @@ -1,83 +0,0 @@ -enhancement: -- description: | - *From GEOIPS#627: 2024-05-31, Add functionality to the CLI which prints out a tree of available commands* - - The GeoIPS CLI provides a variety of commands which aren't necessarily easily exposed - via ``geoips -h``. To improve this issue, we've added a ``geoips tree`` command which - exposes all GeoIPS CLI commands in a tree-like fashion. This way, we can expose all - commands that are available via the GeoIPS CLI, and expose the depth in which these - commands exist. - - By displaying the commands in a depthwise structure, users can understand what commands - are available and how they are called. - - If you just call ``geoips tree``, you'll get the full command tree in a non-colored, - verbose output. - - The output of running ``geoips tree`` is shown below. - - .. code-block:: bash - - geoips tree - - geoips - geoips config - geoips config install - geoips get - geoips get family - geoips get interface - geoips get package - geoips get plugin - geoips list - geoips list interface - geoips list interfaces - geoips list packages - geoips list plugins - geoips list scripts - geoips list test-datasets - geoips list unit-tests - geoips run - geoips run single_source - geoips run data_fusion - geoips run config_based - geoips test - geoips test linting - geoips test script - geoips tree - geoips validate - - ``geoips tree`` additionaly provides optional arguments to filter the output of this - command. Shown below are these optional arguments and descriptions of what each argument - does. - - * ``--color`` - - * The output of ``geoips tree`` might be a little hard to interpret. If you want the - output of ``geoips tree`` to be colored by depth, make sure to use the ``--color`` - flag. (Defaults to False) - - * ``--max-depth`` - - * How many levels of the tree we'd like to expose. Defaults to two levels, which is - shown above. - - * ``--short-name`` - - * The output of ``geoips tree`` provides the full command string at each level. If you - just want the literal command name and every level, make sure to provide this flag. - (Defaults to False) - related-issue: - number: 627 - repo_url: 'https://github.com/NRLMMD-GEOIPS/geoips/' - title: 'New ``geoips tree`` command' - files: - added: - - geoips/commandline/geoips_tree.py - - tests/unit_tests/commandline/test_geoips_tree.py - modified: - - docs/source/userguide/command_line.rst - - geoips/commandline/ancillary_info/cmd_instructions.yaml - - geoips/commandline/commandline_interface.py - - geoips/commandline/geoips_command.py - - geoips/filenames/base_paths.py - - tests/unit_tests/commandline/cli_top_level_tester.py diff --git a/docs/source/releases/latest/636-merge-create_sector_image-functionality-into-CLI.yaml b/docs/source/releases/latest/636-merge-create_sector_image-functionality-into-CLI.yaml deleted file mode 100644 index 5d6b46381..000000000 --- a/docs/source/releases/latest/636-merge-create_sector_image-functionality-into-CLI.yaml +++ /dev/null @@ -1,32 +0,0 @@ -Refactoring Updates: -- description: | - *From GEOIPS#636: 2024-06-05, Merge create_sector_image.py functionality into the CLI* - - This update moves the functionality from the independent console script - ``create_sector_image`` onto the CLI under the command ``geoips test sector``. Since the - CLI has been added to main, we are slowly consolidating all independent console scripts - onto the CLI, so we have a uniform was of executing console commands with GeoIPS. - - There was a slight modification to this command, as we now only allow one sector to - be provided to this command instead of a list of sectors. This fits better with the - nomenclature of the command and can be ran multiple times if the user wants to create - multiple sector images. - - Unit tests have been added for this command as well. - related-issue: - number: 636 - repo_url: 'https://github.com/NRLMMD-GEOIPS/geoips/' - title: 'Merged ``create_sector_image`` into the CLI' - files: - added: - - tests/unit_tests/commandline/test_geoips_test_sector.py - - docs/source/images/command_line_examples/canada.png - deleted: - - geoips/commandline/create_sector_image.py - modified: - - geoips/commandline/ancillary_info/cmd_instructions.yaml - - geoips/commandline/geoips_test.py - - pyproject.toml - - tests/scripts/console_script_create_sector_image.sh - - docs/source/userguide/command_line.rst - - docs/source/userguide/plugin_development/static_sector.rst diff --git a/docs/source/releases/latest/637-use-parse-known-args-to-apply-scope-to-CLI-arguments.yaml b/docs/source/releases/latest/637-use-parse-known-args-to-apply-scope-to-CLI-arguments.yaml deleted file mode 100644 index c59eeca82..000000000 --- a/docs/source/releases/latest/637-use-parse-known-args-to-apply-scope-to-CLI-arguments.yaml +++ /dev/null @@ -1,28 +0,0 @@ -Refactoring Updates: -- description: | - *From GEOIPS#637: 2024-06-06, Using parser.parse_known_args in the CLI to apply scope to arguments* - - While this PR ended up not making use of ``parse_known_args`` (It was buggy and - resulted in overly complex conditionals), we did end up finding a way to apply scope - (ie. share arguments from parents to children) to CLI commands to reduce repeated - portions of the code. This is also nice because we only have to make code changes to one - place if we want to alter arguments that are shared by various commands. - - To do this, we created a ``ParentParsers`` Object in - ``geoips.commandline.geoips_command`` which contains argument parsers that add arguments - which will be shared by some, if not all of the child command classes. For example, the - ``geoips_parser`` attribute of ``ParentParsers`` will be shared to all child commands. - This is because every command is a child of ``geoips`` (the name of ``GeoipsCLI`` class). - A similar ideology applies to ``list_parser``, except that only children of ``list``, - ie. ``GeoipsListPackages``, ``GeoipsListPlugins``, etc. will get the arguments created - by ``list_parser``. This allows for us to share universal arguments such as log level - and command specific argumetns such as ``package_name``. - related-issue: - number: 637 - repo_url: 'https://github.com/NRLMMD-GEOIPS/geoips/' - title: 'Applied Scope to CLI Arguments' - files: - modified: - - geoips/commandline/commandline_interface.py - - geoips/commandline/geoips_command.py - - tests/unit_tests/commandline/cli_top_level_tester.py diff --git a/docs/source/releases/latest/665-issue-with-install-on-debian.yaml b/docs/source/releases/latest/665-issue-with-install-on-debian.yaml deleted file mode 100644 index 32c20e781..000000000 --- a/docs/source/releases/latest/665-issue-with-install-on-debian.yaml +++ /dev/null @@ -1,16 +0,0 @@ -bug fix: -- description: | - When installing GeoIPS on a machine with only python3 (and not python) - we get ``WARNING: 'python --version' failed, please install python >= 3.9 before proceeding`` - because the code looks for ``python`` explicitly. Some Linux distributions decided during - the transition from Python 2 to Python 3 that python should always refer to Python 2, - and the command to run Python 3 would be python3 with a 3 at the end. Debian and Ubuntu did this. - This change adds a message to users informing them how they can change their - system so calls to ``python`` are re-routed to their local ``python3``. - files: - modified: - - 'setup/check_system_requirements.sh' - related-issue: - number: 665 - repo_url: '' - title: 'Fix install on contemporary Debian machines (Ubuntu, etc.)' diff --git a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-2.yaml b/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-2.yaml deleted file mode 100644 index 2f05b8b89..000000000 --- a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-2.yaml +++ /dev/null @@ -1,70 +0,0 @@ -bug-fix: -- description: | - *From GEOIPS#685: 2024-07-17, Fix unit test that will fail if any plugin packages with console scripts are installed* - - There are a few lingering bugs that need fixed before the workshop starts in August. - These were relatively simple to fix so I merged three bug fixes into this PR. - - Bug #1 managed to stay in GeoIPS for a while, and resulted in duplicate log statements - for every log level. This was caused by ``geoips.commandline.log_setup:setup_logging`` - and would result in ``LOG X (num times setup_logging called)`` per a program's execution. - To fix this, I added two global variables in ``setup_logging``, once of which tests if - that function has already been called, the second being a ``log`` variable that will be - returned the first time, and every time after this function is called. With this update, - we need to make sure that the lowest log-level requested should be called FIRST. - Otherwise, it will be obfuscated by higher level log levels. - - Bug #2 was a simple fix, and was just a version change for Cartopy in pyproject.toml. - There was a minor pixel difference in some tests outputs using version 0.22.0, so we - updated this to gt=0.23.0. - - Bug #3 was a possible bug that was introduced by the expose command. There was a - hardcoded portion of a unit test for that command that could result in failed tests - if certain packages had console scripts. We've refactored this command to be dynamic, - and now should pass no matter what packages are provided. - - Bug #4 was related to commandline instructions unit tests, specifically the tests - that checked whether or not a file was newer than another file. These work locally, - but git causes problems with the files write time and these tests sometimes fail. - Since we are confident in the functionality of this code, we've decided to remove - these unit tests for the time being. - title: 'Fix duplicate logs, cartopy version, and expose unit test' - related-issue: | - *From GEOIPS#685: 2024-07-17, Fix unit test that will fail if any plugin packages with console scripts are installed* - files: - modified: - - geoips/commandline/commandline_interface.py - - geoips/commandline/log_setup.py - - pyproject.toml - - tests/unit_tests/commandline/test_expose.py - - tests/unit_tests/commandline/test_get_commandline_instructions.py - deleted: - - tests/unit_tests/commandline/cmd_instructions/json_newer/* - - tests/unit_tests/commandline/cmd_instructions/yaml_newer/* -installation: -- description: | - *From GEOIPS#666: 2024-07-17, Final bug fixes prior to workshop* - - Remove all references to template_fusion_plugin. No longer supporting - template_fusion_plugin - only template_basic_plugin with very basic - plugins, and geoips_plugin_example with more extensive examples. - title: "Remove references to template_fusion_plugin from full install and test" - related-issue: - number: 666 - files: - modified: - - "tests/integration_tests/full_install.sh" - - "tests/integration_tests/full_test.sh" -- description: | - *From GEOIPS#666: 2024-07-17, Final bug fixes prior to workshop* - - Add base and full environment dumps from version 1.13.0. - title: "Add base and full pip and mamba environment files" - related-issue: - number: 666 - files: - modified: - - environments/mamba_base_package_list_1.13.0_20240713.yml - - environments/mamba_full_package_list_1.13.0_20240717.yml - - environments/pip_base_requirements_1.13.0_20240713.txt - - environments/pip_full_requirements_1.13.0_20240717.txt diff --git a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-3.yaml b/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-3.yaml deleted file mode 100644 index 8301e8f75..000000000 --- a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-3.yaml +++ /dev/null @@ -1,10 +0,0 @@ -testing: -- description: | - Make pytest_long called via "all_test_data" vs "all" - files: - modified: - - 'tests/utils/check_code.sh' - related-issue: - number: 666 - repo_url: 'https://github.com/NRLMMD-GEOIPS/geoips' - title: 'Call pytest_long from check_code.sh via all_test_data vs all' diff --git a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-4.yaml b/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-4.yaml deleted file mode 100644 index ab35fc4ba..000000000 --- a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -bug fix: -- description: | - Update build_docs.sh to call brassy to auto-generate the release notes prior - to doc build. - - Also from doc build, call new script to update the release note index to add a - "latest" section if there are latest YAML release notes. We may want to use this - script to add versioned sections to index.rst in the future as well, but - for now it only auto-adds latest. - files: - modified: - - '.gitignore' - - 'docs/build_docs.sh' - - 'docs/source/releases/index.rst' - added: - - 'docs/update_release_note_index.py' - deleted: - - 'docs/source/releases/latest.rst' - related-issue: - number: 666 - repo_url: 'https://github.com/NRLMMD-GEOIPS/geoips' - title: 'Update build_docs.sh to auto-generate .rst with brassy' diff --git a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-5.yaml b/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-5.yaml deleted file mode 100644 index 9fca9a68c..000000000 --- a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-5.yaml +++ /dev/null @@ -1,14 +0,0 @@ -bug fix: -- description: | - Add pip install of plugin package to build_docs.sh to ensure package is - installed before attempting to build API docs. - - Currently, build_docs.sh checks is the current package is installed, and if not - it pip installs it. In the future will likely just error, and assume the - package is installed already if attempting to build documentation. - files: - modified: - - 'docs/build_docs.sh' - related-issue: - number: 666 - title: 'Add pip install of plugin package to build_docs.sh' diff --git a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-6.yaml b/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-6.yaml deleted file mode 100644 index 2069b2758..000000000 --- a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-6.yaml +++ /dev/null @@ -1,11 +0,0 @@ -bug fix: -- description: | - Ensure we capture return value from import mypkgname correctly - when determining if plugin package is installed. Previously - always failed, so always attempted to reinstall the plugin package. - files: - modified: - - 'docs/build_docs.sh' - related-issue: - number: 666 - title: 'Ensure return value correctly captured when importing package in build_docs.sh' diff --git a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-7.yaml b/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-7.yaml deleted file mode 100644 index 9b76e30bc..000000000 --- a/docs/source/releases/latest/666-final-bug-fixes-prior-to-workshop-7.yaml +++ /dev/null @@ -1,11 +0,0 @@ -bug fix: -- description: | - Revert index.rst back to original contents after build_docs.sh completes. - Since the intent is for index.rst to always be auto updated, and never manually - edited, we will ensure all changes are reverted after build_docs completes. - files: - modified: - - 'docs/build_docs.sh' - related-issue: - number: 666 - title: 'Revert docs/source/releases/index.rst back after build_docs.sh' diff --git a/docs/source/releases/latest/679-add-brassy-to-ci.yaml b/docs/source/releases/latest/679-add-brassy-to-ci.yaml deleted file mode 100644 index c7f3ae885..000000000 --- a/docs/source/releases/latest/679-add-brassy-to-ci.yaml +++ /dev/null @@ -1,5 +0,0 @@ -CI: -- description: "Added brassy builds to the CI" - title: "Added brassy to CI" - files: - modified: [".github/workflows/doc-lint-test.yaml"] diff --git a/docs/source/releases/latest/687-change-release-note-yaml-folder-from-version-to-latest.yaml b/docs/source/releases/latest/687-change-release-note-yaml-folder-from-version-to-latest.yaml deleted file mode 100644 index 06336a2fa..000000000 --- a/docs/source/releases/latest/687-change-release-note-yaml-folder-from-version-to-latest.yaml +++ /dev/null @@ -1,18 +0,0 @@ -continuous_integration: -- description: | - Moved the yaml release files from ``docs/source/release/v(version number)/*`` to ``docs/source/release/latest/*``, which now builds to ``latest.rst``. - Added ``latest.rst`` because brassy does not automatically build .rst files at the moment. - Ideally, brassy would create ``latest.rst``, which could be pulled down and built into the docs locally. - Without ``latest.rst`` the docs will not build. So adding a blank file serves as a placeholder until the CI automatically builds and commits a ``latest.rst`` file. - Added latest to ``/docs/source/release/index.rst`` so docs build. - The release note not edited check SHOULD NOT PASS, because.... it was edited 😄 - files: - added: - - 'docs/source/releases/latest.rst' - - 'docs/source/releases/latest/687-change-release-note-yaml-folder-from-version-to-latest.yaml' - modified: - - '.github/workflows/doc-lint-test.yaml' - - 'docs/source/releases/index.rst' - related-issue: - number: 687 - title: 'Change release note .yaml folder from version to latest' diff --git a/docs/source/releases/latest/694-tests-dont-run-if-pr-points-to-a-branch-other-than-main.yaml b/docs/source/releases/latest/694-tests-dont-run-if-pr-points-to-a-branch-other-than-main.yaml deleted file mode 100644 index a5dd01fdd..000000000 --- a/docs/source/releases/latest/694-tests-dont-run-if-pr-points-to-a-branch-other-than-main.yaml +++ /dev/null @@ -1,107 +0,0 @@ -bug fix: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -continuous integration: -- description: | - Update the workflow to run tests regardless of which branch a PR points to. - Previously had only run if pointing to ``main``. - files: - added: - - '' - deleted: - - '' - modified: - - '.github/workflows/doc-lint-test.yaml' - moved: - - '' - related-issue: - number: 694 - repo_url: '' - title: 'Fix the workflow to run on all PRs' -deprecation: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -documentation: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -enhancement: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -performance: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -removal: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' diff --git a/docs/source/releases/latest/698-install-packages-prior-to-attempting-to-build-their-documentation.yaml b/docs/source/releases/latest/698-install-packages-prior-to-attempting-to-build-their-documentation.yaml deleted file mode 100644 index 9af62545c..000000000 --- a/docs/source/releases/latest/698-install-packages-prior-to-attempting-to-build-their-documentation.yaml +++ /dev/null @@ -1,112 +0,0 @@ -bug fix: -- description: | - Add a step in the ``build_sphinx_html`` CI job to install the package for which we - want to build documentation prior to building the documentation. Also remove the - installation code from the ``build_docs.sh`` script. - - This fixes an issue where installation doesn't always happen correctly causing the - docs to fail to build. - files: - added: - - '' - deleted: - - '' - modified: - - 'docs/build_docs.sh' - - '.github/workflows/doc-lint-test.yaml' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: 'Install packages prior to attempting to build their documentation' -continuous integration: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -deprecation: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -documentation: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -enhancement: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -performance: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -removal: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' diff --git a/docs/source/releases/latest/701-change-release-note-check-to-never-allow-any-release-note-rst-files-to-be-edited-locally.yaml b/docs/source/releases/latest/701-change-release-note-check-to-never-allow-any-release-note-rst-files-to-be-edited-locally.yaml deleted file mode 100644 index a90e6dcb4..000000000 --- a/docs/source/releases/latest/701-change-release-note-check-to-never-allow-any-release-note-rst-files-to-be-edited-locally.yaml +++ /dev/null @@ -1,14 +0,0 @@ -continuous integration: -- description: | - Changed boolean check for whether or not release notes have been added - to fix bug where the check for nesessary yaml files would pass - unexpectedly. Additionally, changed check to prevent manual editing of - release files to fail on changes to any `*.rst` files in - `docs/source/release/` instead of just `latest.rst` in the same path. - files: - modified: - - '.github/workflows/doc-lint-test.yaml' - related-issue: - number: 701 - repo_url: '' - title: 'Fix CI issues with incorrectly passing/failing release notes' diff --git a/docs/source/releases/latest/716-check_new_release_note-job-fails-after-merge-to-main.yaml b/docs/source/releases/latest/716-check_new_release_note-job-fails-after-merge-to-main.yaml deleted file mode 100644 index a55698522..000000000 --- a/docs/source/releases/latest/716-check_new_release_note-job-fails-after-merge-to-main.yaml +++ /dev/null @@ -1,105 +0,0 @@ -bug fix: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -continuous integration: -- description: 'Disable check_new_release_note on ``main`` because it compares against ``main``... and thus always fails.' - files: - added: - - '' - deleted: - - '' - modified: - - '.github/workflows/doc-lint-test.yaml' - moved: - - '' - related-issue: - number: 715 - repo_url: '' - title: 'Disable check_new_release_note on ``main``' -deprecation: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -documentation: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -enhancement: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -performance: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -removal: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' diff --git a/docs/source/releases/latest/764-integration-tests-pass-without-noaa-data.yaml b/docs/source/releases/latest/764-integration-tests-pass-without-noaa-data.yaml new file mode 100644 index 000000000..b7ec27ac1 --- /dev/null +++ b/docs/source/releases/latest/764-integration-tests-pass-without-noaa-data.yaml @@ -0,0 +1,12 @@ +bug fix: +- description: | + Removed NOAA AWS dataset from the base tests + because it was unused resulting in a ~6X faster install + (2m 28s -> 24s) and a ~4.25 smaller data footprint (3.8Gb -> 894Mb). + files: + modified: + - 'tests/integration_tests/base_install.sh' + related-issue: + number: 764 + repo_url: '' + title: 'Remove unused NOAA AWS dataset from base tests' diff --git a/docs/source/releases/latest/CLI-documentation-update-for-workshop.yaml b/docs/source/releases/latest/CLI-documentation-update-for-workshop.yaml deleted file mode 100644 index 12c557294..000000000 --- a/docs/source/releases/latest/CLI-documentation-update-for-workshop.yaml +++ /dev/null @@ -1,29 +0,0 @@ -bug fix: -- description: | - This PR fixed small bugs and formatting inconsistencies with the CLI. Specifically, - this PR addressed the format of the output of 'geoips describe' commands, to make - the keys in the yaml output all be in title format rather than (some_val:) vs. - (Some Val:). In this PR we were working on getting CLI documentation up to date, - and discovered a bug with 'geoips list packages' where the '--columns' flag was not - actually working. This was a simple fix and that functionality is now working as - expected. Unit tests have been updated to address these changes. - files: - modified: - - geoips/commandline/commandline_interface.py - - geoips/commandline/geoips_command.py - - geoips/commandline/geoips_describe.py - - geoips/commandline/geoips_list.py - - tests/unit_tests/commandline/test_geoips_describe_interface.py - - tests/unit_tests/commandline/test_geoips_plugin.py - - tests/unit_tests/commandline/test_geoips_list_packages.py - title: 'CLI Bug Fixes and Formatting Updates' -documentation: -- description: | - This PR updated the CLI documentation to reflect the current state of the CLI code. - Largely, this PR went through the documentation of the CLI commands and made sure - that what was documented matched what would actually happen when that command was - ran. There was a duplicate entry in the documentation that was removed as well. - files: - modified: - - docs/source/userguide/commandl_line.rst - title: 'Updated CLI Documentation to reflect the current state of the CLI' \ No newline at end of file diff --git a/docs/source/releases/latest/add_xarray_dict_family.yaml b/docs/source/releases/latest/add_xarray_dict_family.yaml deleted file mode 100644 index d046f3828..000000000 --- a/docs/source/releases/latest/add_xarray_dict_family.yaml +++ /dev/null @@ -1,172 +0,0 @@ -bug fix: -- description: | - *From GEOIPS/geoips#666: 2024-07-16, Final Bug Fixes Prior to Workshop* - - Primary functional change - - * Only check for percentage pixel diffs with the thresholded call to pixelmatch - * Only include exact check for purposes of reporting and outputing exact diff image. - - A few additional updates to help figure out the best way forward for comparison - - * Return immediately if the arrays are identical (avoid running pixelmatch) - * Ensure we run exact and thresholded pixelmatch if the arrays are not identical - * Output separate exact and thresholded diff images - files: - added: - - '' - deleted: - - '' - modified: - - 'geoips/plugins/modules/output_checkers/image.py' - moved: - - '' - related-issue: - number: 666 - repo_url: '' - title: 'Remove exact comparison failures from image output checker plugin' -- description: | - *From GEOIPS/geoips#666: 2024-07-16, Final Bug Fixes Prior to Workshop* - - Ensure all xarray dict based product families are supported within the single - source procflow, including sectored and unsectored dictionaries of xarrays, - algorithm and no algorithm application, and with and without area defs. Note - ALL are not supported, but better identified what is missing / supported now. - files: - added: - - '' - deleted: - - '' - modified: - - 'geoips/plugins/modules/procflows/single_source.py' - moved: - - '' - related-issue: - number: 666 - repo_url: '' - title: 'Add support for algorithm application for xarray dict based families' -- description: | - *From GEOIPS/geoips#666: 2024-07-16, Final Bug Fixes Prior to Workshop* - - Product families that did not have pre-defined output lists and expected data - passed unmodified from the readers to the output formatters were failing due to - a missing else statement ensuring the list of final_products was set even when - the output file list was not pre-defined. Add else statement to set final_products - to curr_products when output file list not defined. - - This resolved an error with product family xrdict_area_product_to_outlist when - it was requested without sectoring, but it worked if sectoring was requested - (followed a different route through the code for each). - files: - modified: - - 'geoips/plugins/modules/procflows/single_source.py' - related-issue: | - number: 666 - url: GEOIPS/geoips#666 - title: 'Fix typo causing product family xrdict_area_product_to_outlist failure' -- description: | - *From GEOIPS/geoips#666: 2024-07-30, Final Bug Fixes Prior to Workshop* - - This test script unnecessarily requires test data. It was updated to skip - the test if $GEOIPS_TESTDATA_DIR doesn't exist, but then it would still fail if some - test data was available, but not the test data required for the abi reader. - Comment this test out altogether for now - we should create a new one at some - point that does not require test data if we want to test the interactive logs - from a directly imported plugin. - files: - modified: - - 'tests/unit_tests/commandline/test_log_setup.py' - related-issue: | - number: 666 - url: GEOIPS/geoips#666 - title: 'Remove interactive log from plugin unit test' -continuous integration: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -deprecation: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -documentation: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -enhancement: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -performance: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' -removal: -- description: '' - files: - added: - - '' - deleted: - - '' - modified: - - '' - moved: - - '' - related-issue: - number: 0 - repo_url: '' - title: '' diff --git a/docs/source/releases/latest/better-download-test-data.yaml b/docs/source/releases/latest/better-download-test-data.yaml new file mode 100644 index 000000000..9491cbb56 --- /dev/null +++ b/docs/source/releases/latest/better-download-test-data.yaml @@ -0,0 +1,32 @@ +bug fix: +- description: | + Fixes a memory bug by streaming download and decompression instead of storing + in memory. + files: + added: + - 'setup/test-data-urls.yaml' + modified: + - 'setup/check_system_requirements.sh' + - 'setup/download_test_data.py' + related-issue: + number: 756 + repo_url: '' + title: 'Improved Data Download and Memory Management During Full Install' +enhancement: +- description: | + ``download_test_data.py`` now accepts dataset names and URLs, with added features + like optional rich text output. Progress towards integrating more functionality + into the GeoIPS CLI. Moved CIRA download URLs to a YAML file for easier management. + files: + added: + - 'setup/test-data-urls.yaml' + deleted: + - '' + modified: + - 'setup/download_test_data.py' + moved: + - '' + related-issue: + number: 0 + repo_url: '' + title: '' \ No newline at end of file diff --git a/docs/source/releases/latest/bug-fix-for-aws-download-script-and-abi_l2_netcdf-reader.yaml b/docs/source/releases/latest/bug-fix-for-aws-download-script-and-abi_l2_netcdf-reader.yaml deleted file mode 100644 index 0e2317415..000000000 --- a/docs/source/releases/latest/bug-fix-for-aws-download-script-and-abi_l2_netcdf-reader.yaml +++ /dev/null @@ -1,20 +0,0 @@ -bug fix: -- description: | - *From GEOIPS/geoips#666: 2024-07-16, Final Bug Fixes Prior to Workshop* - Minor bug fixes to abi_l2_netcdf reader for masking out bad lat/lon points, and - to download_noaa_aws.sh script that allow the script to download other collections - from AWS (i.e. L2 data) - files: - added: - - '' - deleted: - - '' - modified: - - 'geoips/plugins/modules/readers/abi_l2_netcdf.py' - - 'tests/download_noaa_aws.sh' - moved: - - '' - related-issue: - number: 666 - repo_url: 'https://github.com/NRLMMD-GEOIPS/geoips/' - title: 'Minor bug fixes and updates to AWS download script and abi_l2_netcdf reader' diff --git a/docs/source/releases/latest/documentation-better-release-notes.yaml b/docs/source/releases/latest/documentation-better-release-notes.yaml deleted file mode 100644 index 12039cd2d..000000000 --- a/docs/source/releases/latest/documentation-better-release-notes.yaml +++ /dev/null @@ -1,7 +0,0 @@ -documentation: -- description: | - Added summaries to the minor releases in the docs to make searching for a specific - change easier. - title: 'Add top level descriptions to minor releases' - files: - modified: ["docs/source/releases/index.rst"] diff --git a/docs/source/releases/latest/final-CLI-updates-before-workshop.yaml b/docs/source/releases/latest/final-CLI-updates-before-workshop.yaml deleted file mode 100644 index 99bbd5871..000000000 --- a/docs/source/releases/latest/final-CLI-updates-before-workshop.yaml +++ /dev/null @@ -1,25 +0,0 @@ -Regression Fix: -- description: | - With the addition of Command Class Factories, Shared Arguments, and Aliases, we had - an expected regression that would impact the functionality of the command - 'geoips tree'. While we still may make updates to this command in the future, - regarding its output and how we'd like to unit test it, we now have working in a - much cleaner fashion. This PR addressed removing duplicate entries in the output - of the tree, as well hardcoded some conditionals for corner cases that would cause - duplicate entries do to the structure of aliases. For example, we had to add - conditionals when we encountered 'sector' as a command name, as this is an alias for - 'geoips list sectors' and 'geoips describe sectors', and is the actual command name - of 'geoips test sector'. - - Additionally, we commented out the functionality of 'hyperlinking' the text as the - current documentation is not being built up on GitHub and renders that functionality - useless at the moment. In the future, we expect to bring the hyperlink functionality - back in and update how we retrieve the command name as we change how 'prog' is set - for each commands' subparser attribute. - files: - modified: - - geoips/commandline/ancillary_info/alias_mapping.yaml - - geoips/commandline/geoips_describe.py - - geoips/commandline/geoips_list.py - - geoips/commandline/geoips_tree.py - title: 'Fix Regression of geoips tree command' diff --git a/docs/source/releases/latest/fix-env-var-sub.yaml b/docs/source/releases/latest/fix-env-var-sub.yaml new file mode 100644 index 000000000..2c21248e9 --- /dev/null +++ b/docs/source/releases/latest/fix-env-var-sub.yaml @@ -0,0 +1,9 @@ +bug fix: +- description: 'Change string operations on env var replacement to use pathlib to fix issue with links' + files: + modified: + - 'geoips/geoips_utils.py' + related-issue: + number: 792 + repo_url: '' + title: 'Fix environment variable substitution' diff --git a/docs/source/releases/latest/integrate-new-docs-into-old-docs.yaml b/docs/source/releases/latest/integrate-new-docs-into-old-docs.yaml deleted file mode 100644 index d463bde65..000000000 --- a/docs/source/releases/latest/integrate-new-docs-into-old-docs.yaml +++ /dev/null @@ -1,64 +0,0 @@ -documentation: -- description: 'Removed old files, tidied up prose and integrated new docs into old docs via links/toctrees.' - files: - added: - - 'docs/source/license/index.rst' - - 'docs/source/new-docs/contribute/adding-functionality.rst' - - 'docs/source/new-docs/contribute/code-of-conduct.rst' - - 'docs/source/new-docs/contribute/coding_standards.rst' - - 'docs/source/new-docs/contribute/git-github.rst' - - 'docs/source/releases/latest/integrate-new-docs-into-old-docs.yaml' - modified: - - 'docs/source/_templates/index_PKG.html' - - 'docs/source/contact/aboutus.rst' - - 'docs/source/contact/index.rst' - - 'docs/source/devguide/contributors.rst' - - 'docs/source/devguide/dev_setup.rst' - - 'docs/source/devguide/documentation_strategy.rst' - - 'docs/source/devguide/git_workflow.rst' - - 'docs/source/devguide/index.rst' - - 'docs/source/devguide/software_requirements_specification.rst' - - 'docs/source/devguide/unit_tests.rst' - - 'docs/source/devguide/xarray_standards.rst' - - 'docs/source/geoips_api/index.rst' - - 'docs/source/introduction/description_geoips.rst' - - 'docs/source/introduction/index.rst' - - 'docs/source/new-docs/concepts/functionality/index.rst' - - 'docs/source/new-docs/concepts/scope/index.rst' - - 'docs/source/new-docs/contact.rst' - - 'docs/source/new-docs/homepage.rst' - - 'docs/source/starter/expert_installation.rst' - - 'docs/source/starter/extending.rst' - - 'docs/source/starter/index.rst' - - 'docs/source/starter/installation.rst' - - 'docs/source/starter/mac_installation.rst' - - 'docs/source/starter/starter_examples.rst' - - 'docs/source/starter/windows_installation.rst' - - 'docs/source/userguide/command_line.rst' - - 'docs/source/userguide/geoips_structure.rst' - - 'docs/source/userguide/index.rst' - - 'docs/source/userguide/plugin_development/algorithm.rst' - - 'docs/source/userguide/plugin_development/colormapper.rst' - - 'docs/source/userguide/plugin_development/feature_annotator.rst' - - 'docs/source/userguide/plugin_development/gridline_annotator.rst' - - 'docs/source/userguide/plugin_development/output_formatter.rst' - - 'docs/source/userguide/plugin_development/product.rst' - - 'docs/source/userguide/plugin_development/product_default.rst' - - 'docs/source/userguide/plugin_development/reader.rst' - - 'docs/source/userguide/plugin_development/static_sector.rst' - - 'docs/source/userguide/plugin_extend.rst' - - 'docs/source/userguide/plugin_registries.rst' - deleted: - - 'docs/dev/coding_standards.rst' - - 'docs/source/devguide/build_docs.rst' - - 'docs/source/introduction/conduct.rst' - - 'docs/source/introduction/examples_output.rst' - - 'docs/source/introduction/function_summary.rst' - - 'docs/source/new-docs/contribute/adding-functionality/index.rst' - - 'docs/source/new-docs/contribute/code-of-conduct/index.rst' - - 'docs/source/new-docs/contribute/coding-standards/git-github.rst' - - 'docs/source/userguide/function_list.rst' - related-issue: - number: 0 - repo_url: '' - title: 'Integrate new docs into old docs for workshop' diff --git a/docs/source/releases/latest/mac-image-match-bug.yaml b/docs/source/releases/latest/mac-image-match-bug.yaml deleted file mode 100644 index b5ae246ea..000000000 --- a/docs/source/releases/latest/mac-image-match-bug.yaml +++ /dev/null @@ -1,17 +0,0 @@ -bug fix: - - description: | - While testing Mac for the 2024 tutorial, it was found quite a few tests were - failing due to bad image comparisons. Upon greater inspection, it was found that - these tests were failing only due to a couple of different pixels, which we assume - is due to different backend mechanics depending on your OS. Instead of failing a - test if a single pixel is outside of a certain threshold, we've decided to test an - image's match with another pixel via a group basis. - - Ie. If num_pixels_mismatched_by_threshold / total_num_pixels > x%, then we'll - raise an error. This is a more lenient test, but will still catch differences - between two images. All in all, we wan't to ignore tiny differences between images - and catch the more glaring differences that affect lots of pixels when they occur. - files: - modified: - - geoips/plugins/modules/output_checkers/image.py - title: "Mac Image Match Bug" diff --git a/docs/source/releases/latest/memusg-bug-with-mac.yaml b/docs/source/releases/latest/memusg-bug-with-mac.yaml deleted file mode 100644 index c4fbf3eec..000000000 --- a/docs/source/releases/latest/memusg-bug-with-mac.yaml +++ /dev/null @@ -1,16 +0,0 @@ -bug fix: -- description: | - This branch was needed as there was recently a bug introduced on Mac that would prevent - any use of GeoIPS. This was introduced in geoips/utils/memusg.py, where a Process method - was used that was only available on Linux, FreeBSD, SunOS. The method was - ``Process.cpu_num`` and was used in tracking information about the CPU Count used during - a certain GeoIPS Process. Unfortunately, this is unavailable on Mac and caused bugs when - trying to run GeoIPS. - - To fix this, we added a ``platform.system() == Linux`` check for ``Process.cpu_num`` - calls which ensured that function would only be ran if on a Linux system. Otherwise skip - that call and don't collect inforamation on that variable. - title: 'Fix memusg.py bugs that occur on Mac' - files: - modified: - - geoips/utils/memusg.py diff --git a/docs/source/releases/latest/numpy-standards.yaml b/docs/source/releases/latest/numpy-standards.yaml new file mode 100644 index 000000000..90e459882 --- /dev/null +++ b/docs/source/releases/latest/numpy-standards.yaml @@ -0,0 +1,13 @@ +documentation: +- description: | + Throughout GeoIPS, we have inconsistent coding standards. + The numpy docstring format is an external standard we already use, + and so we want that to be explicit in our documentation. + This adds explicit info on using numpy docstrings, as well as examples. + files: + modified: + - 'docs/source/new-docs/contribute/coding_standards.rst' + related-issue: + number: 502 + repo_url: '' + title: 'Add numpy docstrings and examples to coding standards' \ No newline at end of file diff --git a/docs/source/releases/latest/refactor-base-paths.yaml b/docs/source/releases/latest/refactor-base-paths.yaml new file mode 100644 index 000000000..a7cf0de98 --- /dev/null +++ b/docs/source/releases/latest/refactor-base-paths.yaml @@ -0,0 +1,68 @@ +bug fix: +- description: | + Fixes found issue with make_dirs. Previously, if make_dirs was passed a read only directory (eg. ``/read_only``), it would print: + + .. code-block:: bash + + "/read_only: We thought /read_only did not exist, but then it did. " + "Not trying to make directory", + + This is was for anything that could trigger any of these OS Errors: + + - OSError + - BlockingIOError + - ChildProcessError + - ConnectionError + - BrokenPipeError + - FileNotFoundError + - InterruptedError + - IsADirectoryError + - NotADirectoryError + - PermissionError + - ProcessLookupError + - TimeoutError + + Which is far beyond the FileExistsError that was originally supposed to be caught. + files: + added: + - '' + deleted: + - '' + modified: + - 'geoips/filenames/base_paths.py' + moved: + - '' + related-issue: + number: 0 + repo_url: '' + title: 'Fix dir creation bug on OSError' +documentation: +- description: 'Docstrings for base_paths were out of date. They have now been updated.' + files: + added: + - '' + deleted: + - '' + modified: + - 'geoips/filenames/base_paths.py' + moved: + - '' + related-issue: + number: 0 + repo_url: '' + title: 'Write docstrings for base_paths' +Refactor: +- description: 'Base paths was originally very short, but has been added to over time. The original structure did not scale well. The file original file very difficult to read, comprehend and/or maintain. File has been refactored.' + files: + added: + - '' + deleted: + - '' + modified: + - 'geoips/filenames/base_paths.py' + moved: + - '' + related-issue: + number: 0 + repo_url: '' + title: 'Refactor base_paths' diff --git a/docs/source/releases/v1_12_3.rst b/docs/source/releases/v1_12_3.rst index 1cfc85c29..2b20a9e43 100644 --- a/docs/source/releases/v1_12_3.rst +++ b/docs/source/releases/v1_12_3.rst @@ -8,44 +8,42 @@ Version 1.12.3 (2024-05-15) * Formatting Updates + * Allow setting GEOIPS_LOGGING_LEVEL to override default "interactive" * Allow different colors for realtime and admin operational user setups * Bug fixes * Add geolocation variables to geokompsat reader * Add output of radiance to geokompsat reader * Add geolocation variable masking to AHI reader + * Ensure all plugin packages are tested in YAML plugin validation * Ignore missing import in hrit_reader * Fix xarray_to_xarray family * Update ANCILDATDIR to GEOIPS_ANCILDAT * Appropriately pull resolution from LongName - * Fix versions for xarray, rasterio, and cartopy in pyproject.toml + * Check for GEOIPS_DB_URI when using database plugin * Resolve typos in IR_BD and WV colormappers LOG statements - * Update to full path for test_interfaces call in base_test * Resolve typo in netcdf output checker log_with_emphasis call - * Add current miniconda3 install to path - * Check for GEOIPS_DB_URI when using database plugin * Enhancements * Allow passing pyresample kd_tree arguments to interpolators - * Ensure all plugin packages are tested in YAML plugin validation * Allow replacing fields in product_spec_override * Add geolocation resampling to VIIRS reader * Testing Updates - * Use variables[varname] vs getattr(varname) for reader unit tests - * Update config-based AHI backgrounds for AHI geolocation masking update * Minor updates to geokompsat output image title/labels + * Update config-based AHI backgrounds for AHI geolocation masking update + * Use variables[varname] vs getattr(varname) for reader unit tests + * Update to full path for test_interfaces call in base_test * Installation Updates - * Allow passing arguments to pip - * Add general clone_repo command to check_system_requirements + * Fix versions for xarray, rasterio, and cartopy in pyproject.toml + * Add current miniconda3 install to path * Include mamba.sh setup in geoips_conda_init_setup + * Add general clone_repo command to check_system_requirements + * Allow passing arguments to pip * Output version-based requirements.txt immediately after base install. * Require matplotlib<3.9.0 due to matplotlib.cm.get_cmap deprecation * Limit git to <2.45.0 due to bug breaking git lfs clones -* Release updates - - * Allow setting GEOIPS_LOGGING_LEVEL to override default "interactive" * Documentation Updates * Allow including additional sections in plugin package documentation @@ -105,7 +103,6 @@ Ignore missing import in hrit_reader modified: geoips/plugins/modules/readers/utils/hrit_reader.py - Add geolocation variables to geokompsat reader ---------------------------------------------- @@ -150,6 +147,27 @@ Modify xarray_to_xarray family so that it functions correctly. modified: geoips/plugins/modules/procflows/single_source.py +Resolve typo in netcdf output checker log_with_emphasis call +------------------------------------------------------------ + +Fix typo in log_with_emphasis call in netcdf output checker. Was calling for every +line, updated to append all lines into a list, then a single call to log_with_emphasis. + +:: + + modified: geoips/plugins/modules/output_checkers/netcdf.py + +Resolve typos in IR_BD and WV colormappers LOG statements +--------------------------------------------------------- + +Fix typo in IR_BD.py and WV.py colormappers LOG.info statements - previously +had a variable without a corresponding %s. + +:: + + modified: geoips/plugins/modules/colormappers/visir/IR_BD.py + modified: geoips/plugins/modules/colormappers/visir/WV.py + Enhancements ============ @@ -283,30 +301,6 @@ Standardize xarray variable access. modified: tests/unit_tests_long/plugins/modules/readers/test_readers.py -Fix versions for xarray, rasterio, and cartopy in pyproject.toml ----------------------------------------------------------------- - -Require fixed versions for - -* xarray (knmi scat reader bug), -* rasterio (geotiff_standard bug) -* cartopy (test image output gridlines, few pixels difference) - -:: - - modified: pyproject.toml - -Resolve typos in IR_BD and WV colormappers LOG statements ---------------------------------------------------------- - -Fix typo in IR_BD.py and WV.py colormappers LOG.info statements - previously -had a variable without a corresponding %s. - -:: - - modified: geoips/plugins/modules/colormappers/visir/IR_BD.py - modified: geoips/plugins/modules/colormappers/visir/WV.py - Update to full path for test_interfaces call in base_test --------------------------------------------------------- @@ -319,15 +313,21 @@ a pip install of geoips only. modified: tests/integration_tests/base_test.sh -Resolve typo in netcdf output checker log_with_emphasis call ------------------------------------------------------------- +Installation Updates +==================== -Fix typo in log_with_emphasis call in netcdf output checker. Was calling for every -line, updated to append all lines into a list, then a single call to log_with_emphasis. +Fix versions for xarray, rasterio, and cartopy in pyproject.toml +---------------------------------------------------------------- + +Require fixed versions for + +* xarray (knmi scat reader bug), +* rasterio (geotiff_standard bug) +* cartopy (test image output gridlines, few pixels difference) :: - modified: geoips/plugins/modules/output_checkers/netcdf.py + modified: pyproject.toml Add current miniconda3 install to path -------------------------------------- @@ -339,17 +339,6 @@ during conda env calls. modified: setup/geoips_conda_init_setup -Resolve typos in IR_BD and WV colormappers LOG statements ---------------------------------------------------------- - -Fix typo in IR_BD.py and WV.py colormappers LOG.info statements - previously -had a variable without a corresponding %s. - -:: - - modified: geoips/plugins/modules/colormappers/visir/IR_BD.py - modified: geoips/plugins/modules/colormappers/visir/WV.py - Include mamba.sh setup in geoips_conda_init_setup ------------------------------------------------- diff --git a/docs/source/releases/v1_13_1.rst b/docs/source/releases/v1_13_1.rst index 9d1c9036a..8be4b3925 100644 --- a/docs/source/releases/v1_13_1.rst +++ b/docs/source/releases/v1_13_1.rst @@ -3,11 +3,319 @@ | # # # This source code is protected under the license referenced at | # # # https://github.com/NRLMMD-GEOIPS. -Version 1.13.1 (2024-06-01) -***************************** +Version 1.13.1 (2024-07-07) +*************************** -* Update 'update_this_release_note' -* Add release note for v1.13.1 +* Enhancements + + * Add histogram_csv output formatter + * Add oceansat-3 products + * Add get_cmap_data_range to products/dev.py + * Add scipy griddata alpha masking corrections +* Bug Fixes + + * Replace test_data_scat_1.11.2 and 1.11.3 with test_data_scat in unit test + * Update 89pct colormapper data range to match algorithm + * Fixed log double printing +* Installation Updates + + * Add git lfs install to config_geoips + * Add source gitconfigs to base_install + * Freeze git<2.45 and git-lfs<3.4 + * Ensure github downloads also decompress test data + * Allow test_data_github clones with only \*.tgz files + * Freeze netCDF4 < 1.7 due to ABI reader core dump +* Documentation Updates + + * Only required _api and releases docs/source directories + * Require mistune==0.8.4 + * Remove check_old_docs_frozen.py, use git in actions +* Testing Updates + + * Use enviroment variable to control output checker image comparison threshold. + * If tests pass within threshold but not exactly, include exact diff image in logs + * Add bio012024.dat B-deck file + * Remove template_fusion_plugin test calls + * Update test outputs due to minor pixel differences + * Pass clobber=True into atms netcdf test script +* Release Updates + + * Update 'update_this_release_note' + * Add release note for v1.13.1 + * Add release note for v1.13.2, finalize 1.13.1 release + +Enhancements +============ + +Add scipy griddata alpha masking corrections +-------------------------------------------- + +Per recommendations from Nick Zorn MIT LL, implement alpha masking corrections +to help resolve edge of scan issues when using scipy's griddata interpolation + +:: + + modified: geoips/plugins/modules/interpolators/utils/interp_scipy.py + modified: pyproject.toml + +Add get_cmap_data_range to products/dev.py +------------------------------------------ + +To allow pulling ranges separately from algorithm and colormapper plugins, +add an option to geoips/dev/product.py to get the colormapper range. + +This should eventually be added directly to the plugin interfaces, but for now +following the template of get_data_range. + +:: + + modified: geoips/dev/product.py + +Add oceansat-3 products +----------------------- + +Adding sectored/unsectored products, and adding to scat knmi winds. + +:: + + modified: geoips/plugins/modules/readers/scat_knmi_winds_netcdf.py + modified: geoips/plugins/yaml/products/oscat.yaml + +Add histogram_csv output formatter +---------------------------------- + +This was inadvertently not added. + +:: + + new file: geoips/plugins/modules/output_formatters/histogram_csv.py + +Bug Fixes +========= + +Update 89pct colormapper data range to match algorithm +------------------------------------------------------ + +:: + + modified: geoips/plugins/yaml/product_defaults/pmw_89/89pct.yaml + +Replace test_data_scat_1.11.2 and 1.11.3 with test_data_scat in unit test +------------------------------------------------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +:: + + modified: tests/unit_tests/commandline/cli_top_level_tester.py + +Fixed log printing double +------------------------- + +Fixed a bug where the new CLI and the old CLI would both create handlers for logging, +resulting in a double printing for stdout logs, and double lines when running +run_procflow. This was fixed by checking for multiple handlers in logging. + +:: + + modified: geoips/commandline/log_setup.py + +Installation Updates +==================== + +Freeze netCDF4 < 1.7 due to ABI reader core dump +------------------------------------------------ + +*From GEOIPS#513: 2024-06-01, bug fixes* + +ABI Reader core dumps with netCDF 1.7.1. 1.6 still works. + +:: + + modified: pyproject.toml + +Allow test_data_github clones with only \*.tgz files +---------------------------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +Previously required data, outputs, or docs subdir. Allow only a tgz file. + +:: + + modified: setup/check_system_requirements.sh + +Ensure github downloads also decompress test data +------------------------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +Many github repos contain compressed datasets - ensure +data is decompressed after cloning if required. + +:: + + modified: setup/check_system_requirements.sh + +Freeze git<2.45 and git-lfs<3.4 +-------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +Now git and git lfs both break git lfs clones. +Freeze at versions 2.44 and 3.3 until they fix +their bugs. + +:: + + doc/source/starter/installation.rst + +Add git lfs install to config_geoips +------------------------------------ + +*From GEOIPS#513: 2024-06-01, bug fixes* + +:: + + modified: setup/config_geoips + +Add source gitconfigs to base_install +------------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +This includes timeout and buffer size updates that +can help with large clones. + +:: + + modified: setup/check_system_requirements.sh + modified: tests/integration_tests/base_install.sh + +Documentation Updates +===================== + +Remove check_old_docs_frozen.py, use git in actions +--------------------------------------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +:: + + removed: docs/check_old_docs_frozen.py + modified: .github/workflows/check-old-docs-frozen.yml + +Only required _api and releases docs/source directories +------------------------------------------------------------ + +*From GEOIPS#513: 2024-06-01, bug fixes* + +Make introduction and userguide optional. + +:: + + modified: docs/build_docs.sh + modified: docs/source/_templates/index_PKG.html + +Require mistune==0.8.4 +---------------------- + +*From GEOIPS#513: 2024-06-01, bug fixes* + +Somehow mistune 3.0.2 was installed once, which had errors + +m2r2 = { version = "*", optional = true } +Version 3.0.2 causes error , 0.8.4 works +Unsure why there was such a wide range of version numbers installed. + +:: + + File "lib/python3.10/site-packages/m2r2.py", line 82, in + class RestBlockGrammar(mistune.BlockGrammar): + AttributeError: module 'mistune' has no attribute 'BlockGrammar' + +:: + + modified: pyproject.toml + +Testing Updates +=============== + +Use enviroment variable to control output checker image comparison threshold +---------------------------------------------------------------------------- + +Set OUTPUT_CHECKER_THRESHOLD_IMAGE within base_paths - default to 0.05 if env +variable is not set, else use environemnt variable. Default to base_paths +threshold within the image output checker. + +:: + + modified: geoips/filenames/base_paths.py + modified: geoips/interfaces/module_based/output_checkers.py + modified: geoips/plugins/modules/output_checkers/image.py + +If tests pass within threshold but not exactly, include exact diff image in logs +-------------------------------------------------------------------------------- + +If the test passes the threshold, but images are not identical, include the +exact difference in the log output for reference. This allows tracking what +is actually different, but still allows the test to pass. + +:: + + modified: geoips/interfaces/module_based/output_checkers.py + +Pass clobber=True into atms netcdf test script +---------------------------------------------- + +When running atms netcdf output test script for a second time without setting +clobber=True, the output does not get written so the test fails. Update netcdf +output formatters to take a "clobber" argument, then add that argument to the +output_formatter_kwargs in the atms test script. + +:: + + modified: geoips/plugins/modules/output_formatters/netcdf_geoips.py + modified: geoips/plugins/modules/output_formatters/netcdf_xarray.py + modified: tests/scripts/atms.tc.165H.netcdf_geoips.sh + +Update test outputs due to minor pixel differences +-------------------------------------------------- + +A few coastline pixels difference for ABI and AMSR annotated test outputs. Unsure +what caused this, but updating to get 0 return. + +:: + + modified: tests/outputs/abi.static.Infrared.imagery_annotated/20200918.195020.goes-16.abi.Infrared.goes_east.45p56.noaa.10p0.png + modified: tests/outputs/abi.static.Visible.imagery_annotated/20200918.195020.goes-16.abi.Visible.goes_east.41p12.noaa.10p0.png + modified: tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png + modified: tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png + modified: tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png + modified: tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png + modified: tests/outputs/amsr2.tc.89H-Physical.imagery_annotated/20200518_073601_IO012020_amsr2_gcom-w1_89H-Physical_140kts_100p00_res1p0-cr300.png + modified: tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgInfrared-Gray.png + modified: tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgVisible.png + modified: tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgInfrared-Gray.png + modified: tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgVisible.png + +Remove template_fusion_plugin test calls +---------------------------------------- + +Going to rely on a combination of template_basic_plugin, geoips_plugin_examples, +and data_fusion repos for use cases and examples. Only maintain a single plugin +template repo. + +:: + + modified: tests/integration_tests/full_test.sh + +Add bio012024.dat B-deck file +----------------------------- + +:: + + new file: tests/sectors/tc_bdecks/bio012024.dat Release Process =============== @@ -25,6 +333,19 @@ this release note. modified: docs/source/releases/v1_13_1.rst modified: docs/source/releases/index.rst +Add release note for v1.13.2, finalize 1.13.1 release +----------------------------------------------------- + +*From GEOIPS#513: 2024-07-07, 1.13.1 updates* + +All updates until the next release (v1.13.2) will be included in +this release note. This finalizes the 1.13.1 release. + +:: + + modified: docs/source/releases/v1_13_2.rst + modified: docs/source/releases/index.rst + Update 'update_this_release_note' --------------------------------- diff --git a/docs/source/releases/v1_14_0a0.rst b/docs/source/releases/v1_14_0a0.rst index db1f45212..448b68936 100644 --- a/docs/source/releases/v1_14_0a0.rst +++ b/docs/source/releases/v1_14_0a0.rst @@ -3,35 +3,518 @@ | # # # This source code is protected under the license referenced at | # # # https://github.com/NRLMMD-GEOIPS. -Version 1.14.0a0 (2024-07-16) +Version 1.14.0a0 (2024-08-16) ***************************** -* Bug Fixes + * *Bug fix*: Mac Image Match Bug + * *Bug fix*: Update build_docs.sh to auto-generate .rst with brassy + * *Bug fix*: Add pip install of plugin package to build_docs.sh + * *Bug fix*: Minor bug fixes and updates to AWS download script and abi_l2_netcdf reader + * *Bug fix*: Remove exact comparison failures from image output checker plugin + * *Bug fix*: Add support for algorithm application for xarray dict based families + * *Bug fix*: Fix typo causing product family xrdict_area_product_to_outlist failure + * *Bug fix*: Remove interactive log from plugin unit test + * *Bug fix*: Revert docs/source/releases/index.rst back after build_docs.sh + * *Bug fix*: Fix install on contemporary Debian machines (Ubuntu, etc.) + * *Bug fix*: CLI Bug Fixes and Formatting Updates + * *Bug fix*: Install packages prior to attempting to build their documentation + * *Bug fix*: Ensure return value correctly captured when importing package in build_docs.sh + * *Bug fix*: Fix memusg.py bugs that occur on Mac + * *Continuous_integration*: Change release note .yaml folder from version to latest + * *Enhancement*: New ``geoips tree`` command + * *Continuous integration*: Fix CI issues with incorrectly passing/failing release notes + * *Continuous integration*: Disable check_new_release_note on ``main`` + * *Continuous integration*: Fix the workflow to run on all PRs + * *Continuous integration*: Auto-PR built release note on new tag + * *Bug-fix*: Fix duplicate logs, cartopy version, and expose unit test + * *Bug-fix*: Correct typo causing product family xrdict_area_product_to_outlist failure + * *Bug-fix*: Fix failing test data CLI unit tests + * *Installation*: Remove references to template_fusion_plugin from full install and test + * *Installation*: Add base and full pip and mamba environment files + * *Documentation*: Add pinkrst and brassy to dependencies + * *Documentation*: Update LICENSE and DISTRIBUTION to latest approved version + * *Documentation*: Add top level descriptions to minor releases + * *Documentation*: Integrate new docs into old docs for workshop + * *Documentation*: Write System Requirements + * *Documentation*: Updated CLI Documentation to reflect the current state of the CLI + * *Ci*: Added brassy to CI + * *Testing*: Call pytest_long from check_code.sh via all_test_data vs all + * *Testing*: Add NOAA-20, NOAA-21, and NPP downloads to NOAA AWS download script + * *Refactoring updates*: Applied Scope to CLI Arguments + * *Refactoring updates*: Change geoips get to geoips describe + * *Refactoring updates*: Refactored ``geoips list interface`` to use a class factory + * *Refactoring updates*: Refactored ``geoips get `` commands to use a class factory + * *Refactoring updates*: Merged ``create_sector_image`` into the CLI + * *Bug fixes*: Resolve algorithm_colormapper typo in single_source procflow + * *Enchancement*: Add VIIRS HDF5 SDR reader + * *Regression fix*: Fix Regression of geoips tree command + +Bug fix +======= + +Mac Image Match Bug +------------------- + +While testing Mac for the 2024 tutorial, it was found quite a few tests were +failing due to bad image comparisons. Upon greater inspection, it was found that +these tests were failing only due to a couple of different pixels, which we assume +is due to different backend mechanics depending on your OS. Instead of failing a +test if a single pixel is outside of a certain threshold, we've decided to test an +image's match with another pixel via a group basis. + +Ie. If num_pixels_mismatched_by_threshold / total_num_pixels > x%, then we'll +raise an error. This is a more lenient test, but will still catch differences +between two images. All in all, we wan't to ignore tiny differences between images +and catch the more glaring differences that affect lots of pixels when they occur. - * Fix failing test data CLI unit tests - * Fix push of doclinttest-stable image to ghcr.io -* Documentation Updates +:: - * Update LICENSE and DISTRIBUTION to latest approved version, update all file headers - * Add pinkrst and brassy to dependencies + modified: geoips/plugins/modules/output_checkers/image.py -* Enhancements +Update build_docs.sh to auto-generate .rst with brassy +--------------------------------------------------------------- - * Added 'expose' command +Update build_docs.sh to call brassy to auto-generate the release notes prior +to doc build. -Bug Fixes -========= +Also from doc build, call new script to update the release note index to add a +"latest" section if there are latest YAML release notes. We may want to use this +script to add versioned sections to index.rst in the future as well, but +for now it only auto-adds latest. + + +:: + + modified: .gitignore + modified: docs/build_docs.sh + modified: docs/source/releases/index.rst + added: docs/update_release_note_index.py + deleted: docs/source/releases/latest.rst + +Add pip install of plugin package to build_docs.sh +-------------------------------------------------- + +Add pip install of plugin package to build_docs.sh to ensure package is +installed before attempting to build API docs. + +Currently, build_docs.sh checks is the current package is installed, and if not +it pip installs it. In the future will likely just error, and assume the +package is installed already if attempting to build documentation. + + +:: + + modified: docs/build_docs.sh + +Minor bug fixes and updates to AWS download script and abi_l2_netcdf reader +--------------------------------------------------------------------------- + +*From GEOIPS/geoips#666: 2024-07-16, Final Bug Fixes Prior to Workshop* +Minor bug fixes to abi_l2_netcdf reader for masking out bad lat/lon points, and +to download_noaa_aws.sh script that allow the script to download other collections +from AWS (i.e. L2 data) + + +:: + + modified: geoips/plugins/modules/readers/abi_l2_netcdf.py + modified: tests/download_noaa_aws.sh + +Remove exact comparison failures from image output checker plugin +----------------------------------------------------------------- + +*From GEOIPS/geoips#666: 2024-07-16, Final Bug Fixes Prior to Workshop* + +Primary functional change + +* Only check for percentage pixel diffs with the thresholded call to pixelmatch +* Only include exact check for purposes of reporting and outputing exact diff image. + +A few additional updates to help figure out the best way forward for comparison + +* Return immediately if the arrays are identical (avoid running pixelmatch) +* Ensure we run exact and thresholded pixelmatch if the arrays are not identical +* Output separate exact and thresholded diff images + + +:: + + modified: geoips/plugins/modules/output_checkers/image.py + +Add support for algorithm application for xarray dict based families +-------------------------------------------------------------------- + +*From GEOIPS/geoips#666: 2024-07-16, Final Bug Fixes Prior to Workshop* + +Ensure all xarray dict based product families are supported within the single +source procflow, including sectored and unsectored dictionaries of xarrays, +algorithm and no algorithm application, and with and without area defs. Note +ALL are not supported, but better identified what is missing / supported now. + + +:: + + modified: geoips/plugins/modules/procflows/single_source.py + +Fix typo causing product family xrdict_area_product_to_outlist failure +---------------------------------------------------------------------- + +*From GEOIPS/geoips#666: 2024-07-16, Final Bug Fixes Prior to Workshop* + +Product families that did not have pre-defined output lists and expected data +passed unmodified from the readers to the output formatters were failing due to +a missing else statement ensuring the list of final_products was set even when +the output file list was not pre-defined. Add else statement to set final_products +to curr_products when output file list not defined. + +This resolved an error with product family xrdict_area_product_to_outlist when +it was requested without sectoring, but it worked if sectoring was requested +(followed a different route through the code for each). + + +:: + + modified: geoips/plugins/modules/procflows/single_source.py + +Remove interactive log from plugin unit test +-------------------------------------------- + +*From GEOIPS/geoips#666: 2024-07-30, Final Bug Fixes Prior to Workshop* + +This test script unnecessarily requires test data. It was updated to skip +the test if $GEOIPS_TESTDATA_DIR doesn't exist, but then it would still fail if some +test data was available, but not the test data required for the abi reader. +Comment this test out altogether for now - we should create a new one at some +point that does not require test data if we want to test the interactive logs +from a directly imported plugin. + + +:: + + modified: tests/unit_tests/commandline/test_log_setup.py + +Revert docs/source/releases/index.rst back after build_docs.sh +-------------------------------------------------------------- + +Revert index.rst back to original contents after build_docs.sh completes. +Since the intent is for index.rst to always be auto updated, and never manually +edited, we will ensure all changes are reverted after build_docs completes. + + +:: + + modified: docs/build_docs.sh + +Fix install on contemporary Debian machines (Ubuntu, etc.) +---------------------------------------------------------- + +When installing GeoIPS on a machine with only python3 (and not python) +we get ``WARNING: 'python --version' failed, please install python >= 3.9 before proceeding`` +because the code looks for ``python`` explicitly. Some Linux distributions decided during +the transition from Python 2 to Python 3 that python should always refer to Python 2, +and the command to run Python 3 would be python3 with a 3 at the end. Debian and Ubuntu did this. +This change adds a message to users informing them how they can change their +system so calls to ``python`` are re-routed to their local ``python3``. + + +:: + + modified: setup/check_system_requirements.sh + +CLI Bug Fixes and Formatting Updates +------------------------------------ + +This PR fixed small bugs and formatting inconsistencies with the CLI. Specifically, +this PR addressed the format of the output of 'geoips describe' commands, to make +the keys in the yaml output all be in title format rather than (some_val:) vs. +(Some Val:). In this PR we were working on getting CLI documentation up to date, +and discovered a bug with 'geoips list packages' where the '--columns' flag was not +actually working. This was a simple fix and that functionality is now working as +expected. Unit tests have been updated to address these changes. + + +:: + + modified: geoips/commandline/commandline_interface.py + modified: geoips/commandline/geoips_command.py + modified: geoips/commandline/geoips_describe.py + modified: geoips/commandline/geoips_list.py + modified: tests/unit_tests/commandline/test_geoips_describe_interface.py + modified: tests/unit_tests/commandline/test_geoips_plugin.py + modified: tests/unit_tests/commandline/test_geoips_list_packages.py + +Install packages prior to attempting to build their documentation +----------------------------------------------------------------- + +Add a step in the ``build_sphinx_html`` CI job to install the package for which we +want to build documentation prior to building the documentation. Also remove the +installation code from the ``build_docs.sh`` script. + +This fixes an issue where installation doesn't always happen correctly causing the +docs to fail to build. + + +:: + + modified: docs/build_docs.sh + modified: .github/workflows/doc-lint-test.yaml + +Ensure return value correctly captured when importing package in build_docs.sh +------------------------------------------------------------------------------ + +Ensure we capture return value from import mypkgname correctly +when determining if plugin package is installed. Previously +always failed, so always attempted to reinstall the plugin package. + + +:: + + modified: docs/build_docs.sh + +Fix memusg.py bugs that occur on Mac +------------------------------------ + +This branch was needed as there was recently a bug introduced on Mac that would prevent +any use of GeoIPS. This was introduced in geoips/utils/memusg.py, where a Process method +was used that was only available on Linux, FreeBSD, SunOS. The method was +``Process.cpu_num`` and was used in tracking information about the CPU Count used during +a certain GeoIPS Process. Unfortunately, this is unavailable on Mac and caused bugs when +trying to run GeoIPS. + +To fix this, we added a ``platform.system() == Linux`` check for ``Process.cpu_num`` +calls which ensured that function would only be ran if on a Linux system. Otherwise skip +that call and don't collect inforamation on that variable. + + +:: + + modified: geoips/utils/memusg.py + +Continuous_integration +====================== + +Change release note .yaml folder from version to latest +------------------------------------------------------- + +Moved the yaml release files from ``docs/source/release/v(version number)/*`` to ``docs/source/release/latest/*``, which now builds to ``latest.rst``. +Added ``latest.rst`` because brassy does not automatically build .rst files at the moment. +Ideally, brassy would create ``latest.rst``, which could be pulled down and built into the docs locally. +Without ``latest.rst`` the docs will not build. So adding a blank file serves as a placeholder until the CI automatically builds and commits a ``latest.rst`` file. +Added latest to ``/docs/source/release/index.rst`` so docs build. +The release note not edited check SHOULD NOT PASS, because.... it was edited 😄 + + +:: + + added: docs/source/releases/latest.rst + added: docs/source/releases/latest/687-change-release-note-yaml-folder-from-version-to-latest.yaml + modified: .github/workflows/doc-lint-test.yaml + modified: docs/source/releases/index.rst + +Enhancement +=========== + +New ``geoips tree`` command +--------------------------- + +*From GEOIPS#627: 2024-05-31, Add functionality to the CLI which prints out a tree of available commands* + +The GeoIPS CLI provides a variety of commands which aren't necessarily easily exposed +via ``geoips -h``. To improve this issue, we've added a ``geoips tree`` command which +exposes all GeoIPS CLI commands in a tree-like fashion. This way, we can expose all +commands that are available via the GeoIPS CLI, and expose the depth in which these +commands exist. + +By displaying the commands in a depthwise structure, users can understand what commands +are available and how they are called. + +If you just call ``geoips tree``, you'll get the full command tree in a non-colored, +verbose output. + +The output of running ``geoips tree`` is shown below. + +.. code-block:: bash + + geoips tree + + geoips + geoips config + geoips config install + geoips get + geoips get family + geoips get interface + geoips get package + geoips get plugin + geoips list + geoips list interface + geoips list interfaces + geoips list packages + geoips list plugins + geoips list scripts + geoips list test-datasets + geoips list unit-tests + geoips run + geoips run single_source + geoips run data_fusion + geoips run config_based + geoips test + geoips test linting + geoips test script + geoips tree + geoips validate + +``geoips tree`` additionaly provides optional arguments to filter the output of this +command. Shown below are these optional arguments and descriptions of what each argument +does. + +* ``--color`` + + * The output of ``geoips tree`` might be a little hard to interpret. If you want the + output of ``geoips tree`` to be colored by depth, make sure to use the ``--color`` + flag. (Defaults to False) + +* ``--max-depth`` + + * How many levels of the tree we'd like to expose. Defaults to two levels, which is + shown above. + +* ``--short-name`` + + * The output of ``geoips tree`` provides the full command string at each level. If you + just want the literal command name and every level, make sure to provide this flag. + (Defaults to False) + + +:: + + added: geoips/commandline/geoips_tree.py + added: tests/unit_tests/commandline/test_geoips_tree.py + modified: docs/source/userguide/command_line.rst + modified: geoips/commandline/ancillary_info/cmd_instructions.yaml + modified: geoips/commandline/commandline_interface.py + modified: geoips/commandline/geoips_command.py + modified: geoips/filenames/base_paths.py + modified: tests/unit_tests/commandline/cli_top_level_tester.py + +Continuous integration +====================== + +Fix CI issues with incorrectly passing/failing release notes +------------------------------------------------------------ + +Changed boolean check for whether or not release notes have been added +to fix bug where the check for nesessary yaml files would pass +unexpectedly. Additionally, changed check to prevent manual editing of +release files to fail on changes to any `*.rst` files in +`docs/source/release/` instead of just `latest.rst` in the same path. + + +:: + + modified: .github/workflows/doc-lint-test.yaml + +Disable check_new_release_note on ``main`` +------------------------------------------ + +Disable check_new_release_note on ``main`` because it compares against ``main``... and thus always fails. + +:: + + modified: .github/workflows/doc-lint-test.yaml + +Fix the workflow to run on all PRs +---------------------------------- + +Update the workflow to run tests regardless of which branch a PR points to. +Previously had only run if pointing to ``main``. + + +:: + + modified: .github/workflows/doc-lint-test.yaml + +Auto-PR built release note on new tag +------------------------------------- + +Added workflow to build, commit and PR a built release note with brassy + +:: + + added: .github/workflows/new-brassy-note.yaml + +Bug-fix +======= + +Fix duplicate logs, cartopy version, and expose unit test +--------------------------------------------------------- + +*From GEOIPS#685: 2024-07-17, Fix unit test that will fail if any plugin packages with console scripts are installed* + +There are a few lingering bugs that need fixed before the workshop starts in August. +These were relatively simple to fix so I merged three bug fixes into this PR. + +Bug #1 managed to stay in GeoIPS for a while, and resulted in duplicate log statements +for every log level. This was caused by ``geoips.commandline.log_setup:setup_logging`` +and would result in ``LOG X (num times setup_logging called)`` per a program's execution. +To fix this, I added two global variables in ``setup_logging``, once of which tests if +that function has already been called, the second being a ``log`` variable that will be +returned the first time, and every time after this function is called. With this update, +we need to make sure that the lowest log-level requested should be called FIRST. +Otherwise, it will be obfuscated by higher level log levels. + +Bug #2 was a simple fix, and was just a version change for Cartopy in pyproject.toml. +There was a minor pixel difference in some tests outputs using version 0.22.0, so we +updated this to gt=0.23.0. + +Bug #3 was a possible bug that was introduced by the expose command. There was a +hardcoded portion of a unit test for that command that could result in failed tests +if certain packages had console scripts. We've refactored this command to be dynamic, +and now should pass no matter what packages are provided. + +Bug #4 was related to commandline instructions unit tests, specifically the tests +that checked whether or not a file was newer than another file. These work locally, +but git causes problems with the files write time and these tests sometimes fail. +Since we are confident in the functionality of this code, we've decided to remove +these unit tests for the time being. + + +:: + + modified: geoips/commandline/commandline_interface.py + modified: geoips/commandline/log_setup.py + modified: pyproject.toml + modified: tests/unit_tests/commandline/test_expose.py + modified: tests/unit_tests/commandline/test_get_commandline_instructions.py + deleted: tests/unit_tests/commandline/cmd_instructions/json_newer/* + deleted: tests/unit_tests/commandline/cmd_instructions/yaml_newer/* + +Correct typo causing product family xrdict_area_product_to_outlist failure +-------------------------------------------------------------------------- + +Correct typo causing product family xrdict_area_product_to_outlist failure. Product +families that did not have pre-defined output lists and expected data passed +unmodified from the readers to the output formatters were failing due to a missing +else statement ensuring the list of final_products was set even when the output file +list was not pre-defined. Add else statement to set final_products to curr_products +when output file list not defined. This resolved an error with product family +xrdict_area_product_to_outlist when it was requested without sectoring, but it +worked if sectoring was requested (followed a different route through the code for +each). + + +:: + + modified: geoips/plugins/modules/procflows/single_source.py Fix failing test data CLI unit tests ------------------------------------ -*From GEOIPS#666: 2024-07-14, Final Bug Fixes Prior to Workshop* +Fix failing test data CLI unit tests. `test_geoips_config_install.py` and +`test_log_setup.py` had 1 or more tests that were failing, which caused the CI to +fail as well. Fix these unit tests so we can actually use the CI to address problems +that are coming from new PRs. -``test_geoips_config_install.py`` and ``test_log_setup.py`` had 1 or more tests that -were failing, which caused the CI to fail as well. Fix these unit tests so -we can actually use the CI to address problems that are coming from new PRs. Below are -the files that required changes. :: @@ -39,75 +522,438 @@ the files that required changes. modified: tests/unit_tests/commandline/cli_top_level_tester.py deleted: tests/unit_tests/commandline/test_geoips_config_install.py -Fix push of doclinttest-stable image to ghcr.io ------------------------------------------------ +Installation +============ + +Remove references to template_fusion_plugin from full install and test +---------------------------------------------------------------------- -*From GEOIPS#681: 2024-07-16, Fix push of doclinttest-stable image to ghcr.io* +*From GEOIPS#666: 2024-07-17, Final bug fixes prior to workshop* + +Remove all references to template_fusion_plugin. No longer supporting +template_fusion_plugin - only template_basic_plugin with very basic +plugins, and geoips_plugin_example with more extensive examples. -Incorrect/old variables were being used in this step, but the issue wasn't -revealed until we pushed to main because the step didn't run except on main. -Added an additional step to also push doclinttest-latest in the same manner, -but from any branch to ensure that the doclinttest-stable step will succeed. :: - modified: .github/workflows/doc-lint-test.yaml + modified: tests/integration_tests/full_install.sh + modified: tests/integration_tests/full_test.sh + +Add base and full pip and mamba environment files +------------------------------------------------- + +*From GEOIPS#666: 2024-07-17, Final bug fixes prior to workshop* -Documentation Updates -===================== +Add base and full environment dumps from version 1.13.0. + + +:: + + modified: environments/mamba_base_package_list_1.13.0_20240713.yml + modified: environments/mamba_full_package_list_1.13.0_20240717.yml + modified: environments/pip_base_requirements_1.13.0_20240713.txt + modified: environments/pip_full_requirements_1.13.0_20240717.txt + +Documentation +============= Add pinkrst and brassy to dependencies -------------------------------------- -*From GEOIPS#675: 2024-07-16, Add brassy and pinkrst to geoips lint dependencies* +Add pinkrst and brassy to dependencies. + +:: + + modified: pyproject.toml + +Update LICENSE and DISTRIBUTION to latest approved version +---------------------------------------------------------- + +Update LICENSE and DISTRIBUTION to latest approved version. :: - modified: pyproject.toml + modified: LICENSE + modified: DISTRIBUTION -Update LICENSE and DISTRIBUTION to latest approved version, update all file headers ------------------------------------------------------------------------------------ +Add top level descriptions to minor releases +-------------------------------------------- -*From GEOIPS#666: 2024-07-14, Final Bug Fixes Prior to Workshop* +Added summaries to the minor releases in the docs to make searching for a specific +change easier. -This file header should no longer have to change. :: - modified: LICENSE - modified: DISTRIBUTION - modified: Every single file + modified: docs/source/releases/index.rst -Enhancements -============ +Integrate new docs into old docs for workshop +--------------------------------------------- -Added 'expose' command ----------------------- +Removed old files, tidied up prose and integrated new docs into old docs via links/toctrees. + +:: + + added: docs/source/license/index.rst + added: docs/source/new-docs/contribute/adding-functionality.rst + added: docs/source/new-docs/contribute/code-of-conduct.rst + added: docs/source/new-docs/contribute/coding_standards.rst + added: docs/source/new-docs/contribute/git-github.rst + added: docs/source/releases/latest/integrate-new-docs-into-old-docs.yaml + modified: docs/source/_templates/index_PKG.html + modified: docs/source/contact/aboutus.rst + modified: docs/source/contact/index.rst + modified: docs/source/devguide/contributors.rst + modified: docs/source/devguide/dev_setup.rst + modified: docs/source/devguide/documentation_strategy.rst + modified: docs/source/devguide/git_workflow.rst + modified: docs/source/devguide/index.rst + modified: docs/source/devguide/software_requirements_specification.rst + modified: docs/source/devguide/unit_tests.rst + modified: docs/source/devguide/xarray_standards.rst + modified: docs/source/geoips_api/index.rst + modified: docs/source/introduction/description_geoips.rst + modified: docs/source/introduction/index.rst + modified: docs/source/new-docs/concepts/functionality/index.rst + modified: docs/source/new-docs/concepts/scope/index.rst + modified: docs/source/new-docs/contact.rst + modified: docs/source/new-docs/homepage.rst + modified: docs/source/starter/expert_installation.rst + modified: docs/source/starter/extending.rst + modified: docs/source/starter/index.rst + modified: docs/source/starter/installation.rst + modified: docs/source/starter/mac_installation.rst + modified: docs/source/starter/starter_examples.rst + modified: docs/source/starter/windows_installation.rst + modified: docs/source/userguide/command_line.rst + modified: docs/source/userguide/geoips_structure.rst + modified: docs/source/userguide/index.rst + modified: docs/source/userguide/plugin_development/algorithm.rst + modified: docs/source/userguide/plugin_development/colormapper.rst + modified: docs/source/userguide/plugin_development/feature_annotator.rst + modified: docs/source/userguide/plugin_development/gridline_annotator.rst + modified: docs/source/userguide/plugin_development/output_formatter.rst + modified: docs/source/userguide/plugin_development/product.rst + modified: docs/source/userguide/plugin_development/product_default.rst + modified: docs/source/userguide/plugin_development/reader.rst + modified: docs/source/userguide/plugin_development/static_sector.rst + modified: docs/source/userguide/plugin_extend.rst + modified: docs/source/userguide/plugin_registries.rst + deleted: docs/dev/coding_standards.rst + deleted: docs/source/devguide/build_docs.rst + deleted: docs/source/introduction/conduct.rst + deleted: docs/source/introduction/examples_output.rst + deleted: docs/source/introduction/function_summary.rst + deleted: docs/source/new-docs/contribute/adding-functionality/index.rst + deleted: docs/source/new-docs/contribute/code-of-conduct/index.rst + deleted: docs/source/new-docs/contribute/coding-standards/git-github.rst + deleted: docs/source/userguide/function_list.rst + +Write System Requirements +------------------------- + +Added system requirements to the documentation. This includes minimum and recommended +requirements for users and developers as well as language limiting the applicability +of the system requirements. + + +:: -*From GEOIPS#274: 2023-08-19, Add useage to setup.sh for command discovery* + modified: docs/source/new-docs/getting-started/system-requirements/index.rst -The aforementioned issue requested that we add functionality to expose all commands -available in the GeoIPS environment. Rather than adding this functionality to setup.sh, -I've decided to add another function to geoips_utils.py which will expose all GeoIPS -commands in a tabular format. We expose available commands in a certain GeoIPS plugin -package to the terminal for easy perusal by the user. This will work for any new command -added in the future as well. This function adds one new dependency, ``tabulate``, though -``tabulate`` is added as a dependency to the CLI PRs anyways. We expect this might be -refactored into the CLI in a future PR. +Updated CLI Documentation to reflect the current state of the CLI +----------------------------------------------------------------- -To use this new function, reinstall GeoIPS via ``pip install -e .``, and run the command -``expose``. +This PR updated the CLI documentation to reflect the current state of the CLI code. +Largely, this PR went through the documentation of the CLI commands and made sure +that what was documented matched what would actually happen when that command was +ran. There was a duplicate entry in the documentation that was removed as well. -This branch additionally includes a very small change made to two reader files and the -unit test for ``geoips validate`` as there were a couple of unit tests that could fail -due to relative imports used in a non-editable installed package. :: + modified: docs/source/userguide/commandl_line.rst + +Ci +== + +Added brassy to CI +------------------ + +Added brassy builds to the CI + +:: + + modified: .github/workflows/doc-lint-test.yaml + +Testing +======= + +Call pytest_long from check_code.sh via all_test_data vs all +------------------------------------------------------------ + +Make pytest_long called via "all_test_data" vs "all" + + +:: + + modified: tests/utils/check_code.sh + +Add NOAA-20, NOAA-21, and NPP downloads to NOAA AWS download script +------------------------------------------------------------------- + +Add NOAA-20, NOAA-21, and NPP downloads to NOAA AWS download script + +:: + + modified: tests/download_noaa_aws.sh + +Refactoring updates +=================== + +Applied Scope to CLI Arguments +------------------------------ + +*From GEOIPS#637: 2024-06-06, Using parser.parse_known_args in the CLI to apply scope to arguments* + +While this PR ended up not making use of ``parse_known_args`` (It was buggy and +resulted in overly complex conditionals), we did end up finding a way to apply scope +(ie. share arguments from parents to children) to CLI commands to reduce repeated +portions of the code. This is also nice because we only have to make code changes to one +place if we want to alter arguments that are shared by various commands. + +To do this, we created a ``ParentParsers`` Object in +``geoips.commandline.geoips_command`` which contains argument parsers that add arguments +which will be shared by some, if not all of the child command classes. For example, the +``geoips_parser`` attribute of ``ParentParsers`` will be shared to all child commands. +This is because every command is a child of ``geoips`` (the name of ``GeoipsCLI`` class). +A similar ideology applies to ``list_parser``, except that only children of ``list``, +ie. ``GeoipsListPackages``, ``GeoipsListPlugins``, etc. will get the arguments created +by ``list_parser``. This allows for us to share universal arguments such as log level +and command specific argumetns such as ``package_name``. + + +:: + + modified: geoips/commandline/commandline_interface.py + modified: geoips/commandline/geoips_command.py + modified: tests/unit_tests/commandline/cli_top_level_tester.py + +Change geoips get to geoips describe +------------------------------------ + +*From GEOIPS#495: 2024-04-15, Change CLI Command 'geoips get' to 'geoips describe'* + +This update renames all references of 'geoips get' to 'geoips describe'. 'Describe' +is a better name for this command as it generally provides additional information +about a certain GeoIPS artifact, whereas 'get' could be a bit ambiguous as users +may think we are actually getting a certain plugin, interface, etc. Another reason +for this update is that all references to the CLI in the tutorial use +'geoips describe', rather than 'geoips get'. This PR was only a nomenclature change +and did not impact any of the actual functionality of the CLI. + + +:: + + added: geoips/commandline/geoips_describe.py + added: tests/unit_tests/commandline/test_geoips_describe_family.py + added: tests/unit_tests/commandline/test_geoips_describe_interface.py + added: tests/unit_tests/commandline/test_geoips_describe_package.py + added: tests/unit_tests/commandline/test_geoips_describe_plugin.py + deleted: geoips/commandline/geoips_get.py + deleted: tests/unit_tests/commandline/test_geoips_get_family.py + deleted: tests/unit_tests/commandline/test_geoips_get_interface.py + deleted: tests/unit_tests/commandline/test_geoips_get_package.py + deleted: tests/unit_tests/commandline/test_geoips_get_plugin.py + modified: docs/source/userguide/command_line.rst + modified: geoips/commandline/ancillary_info/alias_mapping.yaml + modified: geoips/commandline/ancillary_info/cmd_instructions.yaml + modified: geoips/commandline/commandline_interface.py + modified: geoips/commandline/geoips_command.py + +Refactored ``geoips list interface`` to use a class factory +----------------------------------------------------------- + +*From GEOIPS#576: 2024-05-11, CLI: Reduce number of command layers where possible and appropriate* +*From GEOIPS#573: 2024-05-11, Use class factories to generate subcommands for commands like ``geoips get family`` and ``geoips get interface``* + +While this PR doesn't fix all of the problems addressed in the issues above, it is our +initial attempt at creating a class factory for certain CLI commands to reduce the +verbage needed to execute those commands. Spefically in this branch, we've addressed the +command class ``GeoipsListSingleInterface``. We now use the aforementioned class as a +base class to build ``GeoipsListSingleInterface`` classes at runtime. + +By doing so, we can now run ``geoips list `` rather than +``geoips list interface ``. This improves the readability of this +command and makes it much easier to type via the CLI. We expect in future PRs to address +similar commands, such as ``geoips get family `` and +``geoips get plugin `` for the reasons mentioned +previously. + + +:: + + modified: docs/source/userguide/command_line.rst + modified: geoips/commandline/ancillary_info/cmd_instructions.yaml + modified: geoips/commandline/geoips_command.py + modified: geoips/commandline/geoips_list.py + modified: tests/unit_tests/commandline/cli_top_level_tester.py + modified: tests/unit_tests/commandline/test_geoips_list_interface.py + +Refactored ``geoips get `` commands to use a class factory +-------------------------------------------------------------------------- + +*From GEOIPS#576: 2024-05-11, CLI: Reduce number of command layers where possible and appropriate* +*From GEOIPS#573: 2024-05-11, Use class factories to generate subcommands for commands like ``geoips get family`` and ``geoips get interface``* + +This branch refactors commands which use the form of ``geoips get `` to +make use of class factories. Before, we had to run commands such as: + +* ``geoips get interface `` +* ``geoips get family `` +* ``geoips get plugin `` + +The aforementioned commands were overly verbose and needed some refactoring to reduce +the levels required to execute a certain command. In this branch, we refactored +``GeoipsGetInterface`` to be a base command for ``get ``, +``get family ``, and +``get ``. This resulted in the removal of +``GeoipsGetFamily`` and ``GeoipsGetPlugin`` and the refactoring of +``GeoipsGetInterface`` to include the functionality of those other commands. We can use +class factories for each interface to execute these commands now. + +On top of this, we implemented aliases for a variety of CLI commands. For an exact +listing of aliases available for each command, see +``geoips/commandline/ancillary_info/alias_mapping.yaml``, which includes aliases +supported for each command. Now, we can run a verbose command such as: + +* ``geoips list algorithms -p geoips`` + +and replace it with + +* ``geoips ls algs -p geoips`` + +This also reduces the verbosity of certain CLI commands. Note, the unit tests have been +modified to reflect these changes. + + +:: + + modified: .gitignore modified: pyproject.toml - modified: geoips/errors.py - modified: geoips/geoips_utils.py - modified: geoips/plugins/modules/readers/seviri_hrit.py - modified: geoips/plugins/modules/readers/windsat_remss_winds_netcdf.py - modified: tests/unit_tests/commandline/test_geoips_validate.py - added: tests/unit_tests/commandline/test_expose.py + modified: docs/source/_templates/conf_PKG.py + modified: docs/source/userguide/command_line.rst + modified: geoips/commandline/ancillary_info/cmd_instructions.yaml + modified: geoips/commandline/cmd_instructions.py + modified: geoips/commandline/commandline_interface.py + modified: geoips/commandline/geoips_command.py + modified: geoips/commandline/geoips_get.py + modified: tests/unit_tests/commandline/cli_top_level_tester.py + modified: tests/unit_tests/commandline/test_geoips_get_family.py + modified: tests/unit_tests/commandline/test_geoips_get_interface.py + modified: tests/unit_tests/commandline/test_geoips_get_package.py + modified: tests/unit_tests/commandline/test_geoips_get_plugin.py + modified: tests/unit_tests/commandline/test_geoips_list_interface.py + modified: tests/unit_tests/commandline/test_geoips_list_interfaces.py + modified: tests/unit_tests/commandline/test_geoips_list_packages.py + modified: tests/unit_tests/commandline/test_geoips_list_plugins.py + modified: tests/unit_tests/commandline/test_geoips_list_scripts.py + modified: tests/unit_tests/commandline/test_geoips_list_test_datasets.py + modified: tests/unit_tests/commandline/test_geoips_list_unit_tests.py + modified: tests/unit_tests/commandline/test_get_commandline_instructions.py + added: geoips/commandline/ancillary_info/alias_mapping.yaml + +Merged ``create_sector_image`` into the CLI +------------------------------------------- + +*From GEOIPS#636: 2024-06-05, Merge create_sector_image.py functionality into the CLI* + +This update moves the functionality from the independent console script +``create_sector_image`` onto the CLI under the command ``geoips test sector``. Since the +CLI has been added to main, we are slowly consolidating all independent console scripts +onto the CLI, so we have a uniform was of executing console commands with GeoIPS. + +There was a slight modification to this command, as we now only allow one sector to +be provided to this command instead of a list of sectors. This fits better with the +nomenclature of the command and can be ran multiple times if the user wants to create +multiple sector images. + +Unit tests have been added for this command as well. + + +:: + + added: tests/unit_tests/commandline/test_geoips_test_sector.py + added: docs/source/images/command_line_examples/canada.png + deleted: geoips/commandline/create_sector_image.py + modified: geoips/commandline/ancillary_info/cmd_instructions.yaml + modified: geoips/commandline/geoips_test.py + modified: pyproject.toml + modified: tests/scripts/console_script_create_sector_image.sh + modified: docs/source/userguide/command_line.rst + modified: docs/source/userguide/plugin_development/static_sector.rst + +Bug fixes +========= + +Resolve algorithm_colormapper typo in single_source procflow +------------------------------------------------------------ + +Resolve algorithm_interpolator_colormapper typo in single_source procflow +Pass in xarray.Dataset() rather than alg_xarray as the "interp_xarray" to +the "perform_interpolation" function. Previously the xarray was not being +interpolated, because it thought the alg_xarray was the pre-interpolated array. +This bug was introduced when refactoring single source to avoid +re-interpolating. We should probably add an integration test for this +product family. + + +:: + + modified: geoips/plugins/modules/procflows/single_source.py + +Enchancement +============ + +Add VIIRS HDF5 SDR reader +------------------------- + +Created reader for VIIRS HDF5 SDR data, correcting for bowtie +distortion for single and multiple files. + + +:: + + added: geoips/plugins/modules/readers/viirs_sdr_hdf5.py + +Regression fix +============== + +Fix Regression of geoips tree command +------------------------------------- + +With the addition of Command Class Factories, Shared Arguments, and Aliases, we had +an expected regression that would impact the functionality of the command +'geoips tree'. While we still may make updates to this command in the future, +regarding its output and how we'd like to unit test it, we now have working in a +much cleaner fashion. This PR addressed removing duplicate entries in the output +of the tree, as well hardcoded some conditionals for corner cases that would cause +duplicate entries do to the structure of aliases. For example, we had to add +conditionals when we encountered 'sector' as a command name, as this is an alias for +'geoips list sectors' and 'geoips describe sectors', and is the actual command name +of 'geoips test sector'. + +Additionally, we commented out the functionality of 'hyperlinking' the text as the +current documentation is not being built up on GitHub and renders that functionality +useless at the moment. In the future, we expect to bring the hyperlink functionality +back in and update how we retrieve the command name as we change how 'prog' is set +for each commands' subparser attribute. + + +:: + + modified: geoips/commandline/ancillary_info/alias_mapping.yaml + modified: geoips/commandline/geoips_describe.py + modified: geoips/commandline/geoips_list.py + modified: geoips/commandline/geoips_tree.py diff --git a/docs/source/releases/version_summaries.yaml b/docs/source/releases/version_summaries.yaml new file mode 100644 index 000000000..417190793 --- /dev/null +++ b/docs/source/releases/version_summaries.yaml @@ -0,0 +1,63 @@ +summaries: +- version: 1.12 + summary: | + Added documentation and enhancements for coding standards, fixed bugs in polar data, log functions, and Dockerfile; + Introduced documentation for PEP8, linters, and plugin registries; Refactored optional imports, plugin registries. Fixed + issues with create_plugin_registries, ATMS platform names, and pytest dependencies. +- version: 1.11 + summary: | + Removed flake8_docstring_only flag from test_all.sh and full_test.sh across the repository. Fixed issues in integration + tests and ensured full flake8 tests applied. +- version: 1.10 + summary: | + Updated interface names throughout the code base, moved YAML sector files to new directories within the geoips/plugins + structure, added support for both the old and new dynamic sector YAML formatting, and renamed plugins to enhance + clarity. Introduced a try/except block in sector_utils/tc_tracks.py to support both old and new dynamic sector YAML + formats. Consolidated both dynamic and static sectors under the unified directory structure in + geoips/plugins/yaml/sectors, and renamed various modules to reflect their functionalities more accurately. +- version: 1.9 + summary: | + Removed GEOIPS BASEDIR references and introduced GEOIPS_TESTDATA_DIR and GEOIPS_DEPENDENCIES_DIR in + filenames/base_paths.py, updated and implemented sphinx-apidoc for Sphinx documentation builds, finalized RST + documentation outline to align with SharePoint documentation, replaced setup.py with pyproject.toml for project setup, + added "cbar_tick_labels" and "cbar_spacing" fields in Rain colormap mpl_color_info, allowed processing when no tropical + cyclones are present in the database, installed geoips[doc, test, lint] dependencies in setup.sh, and configured code + style to ignore flake8 W503 due to conflicts with black. +- version: 1.8 + summary: | + Renamed SSMI/S 89GHz products to 91GHz, updated F-16 SSMI/S scaling, resolved basename error, added geoips_clavrx to + test_full_install, and corrected minor sphinx build warnings. Fixed tests, ensured accurate product naming, and improved + installation and documentation processes. +- version: 1.7 + summary: | + Updated matplotlib version dependency, extended SAR incident angle range, and converted CHANGELOG and documentation + formats to RST. Removed outdated sections, fixed minor documentation issues, and improved documentation organization and + building process. +- version: 1.6 + summary: | + Added resource usage statistics, documentation updates, and code style enforcement checks. Fixed numpy builtin aliases, + product family typos, and output requirements. +- version: 1.5 + summary: | + Updated the Python installation process, added alternate command line arguments functions, created unique storm + directory names for invests, and fixed bugs. Expanded test repository updates and improved metadata handling. +- version: 1.4 + summary: | + Updated documentation and installation requirements, added new test scripts and functionality. Fixed real-time + processing and coverage checks. +- version: 1.3 + summary: | + Replaced instances of 'atcf' with 'tc', removed support for GEOIPSFINAL and GEOIPSTEMP, and consolidated directories + into 'geoips_outdirs.' Added dev/title.py interface, updated template pull request ticket, allowed passing + 'title_copyright,' and added new DATABASESUCCESS and DATABASEFAILURE string checks. +- version: 1.2 + summary: | + Updated bdeck parser, SMOS text winds, test_interfaces.py, attribute names, call signatures; added ABI test scripts, + Software Requirements Specification. Replaced original_source_filename with original_source_filenames; simplified setup, + command line args checking, installation steps. +- version: 1.1 + summary: | + Addition of new features like MODIS reader, stitched product output, sector overpass database, and Visible global + stitched. Refactoring for Python 3 operation, and resolving errors in MODIS processing and RGB MTIF color issue. There + are also breaking changes including modularized PMW algorithms, changed dictionary structures, and standardized platform + names. \ No newline at end of file diff --git a/docs/source/starter/installation.rst b/docs/source/starter/installation.rst index d8dff63a4..bd58472ff 100644 --- a/docs/source/starter/installation.rst +++ b/docs/source/starter/installation.rst @@ -127,6 +127,7 @@ and run integration tests: # Ensure geoips python environment enabled + # Install base GeoIPS package and minimal test datasets. $GEOIPS_PACKAGES_DIR/geoips/tests/integration_tests/base_install.sh # Create the GeoIPS plugin registries diff --git a/docs/source/starter/mac_installation.rst b/docs/source/starter/mac_installation.rst index dbddc307c..0d74ba0d3 100644 --- a/docs/source/starter/mac_installation.rst +++ b/docs/source/starter/mac_installation.rst @@ -155,3 +155,4 @@ look something like below, indicating that none of the tests failed: Total run time: 82 seconds Number data types run: 3 Number data types failed: 0 + diff --git a/docs/source/starter/starter_examples.rst b/docs/source/starter/starter_examples.rst index 04815b557..0c9c42da6 100644 --- a/docs/source/starter/starter_examples.rst +++ b/docs/source/starter/starter_examples.rst @@ -14,6 +14,9 @@ available publicly, you can run the test script $GEOIPS_PACKAGES_DIR/geoips/tests/test_full_install.sh. This will download, install, and test all possible data types and products. +Additionally, we provide a helper script ``download_test_data.py`` to download data sets +for expert users. + Step by Step ============ diff --git a/docs/update_release_note_index.py b/docs/update_release_note_index.py index a73a86820..b3d6d749c 100644 --- a/docs/update_release_note_index.py +++ b/docs/update_release_note_index.py @@ -1,63 +1,131 @@ #!/bin/env python + +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + """Update release note index with latest version release reference.""" -from sys import argv, exit - -index_filename = argv[1] -add_version = argv[2] - -# Eventually we will likely support actual versions or latest. For now only -# support "latest". -# This may end up being just auto-generating the ENTIRE index.rst, which will -# remove the need for specifying a version at all. For now we are only adding -# the "latest" section to index.rst so the doc build doesn't fail from a non-existent -# reference. In order to auto-generate the entire index.rst, we would have to -# include the high level minor release version summaries somewhere outside index.rst -# (right now the only specification of the high level summaries is in index.rst, -# so we can't entirely auto-generate it). -# This will be revisited after the 2024 workshop. For now this will work. -if add_version != "latest": - print("Actual versions not yet supported - only latest") - exit(1) - # Something like this for adding actual version to index.rst - # Or we may just auto-generate the entire index.rst - # major, minor, bug_fix = add_version.split(".") - # minor_version = f"{major}.{minor}" - # release_note_version = f"v{add_version.replace(".", "_")}" -with open(index_filename, "r") as fobj: - first = True - new_lines = [] - last_line = None - for line in fobj.readlines(): - # Make sure we write all the lines back out - if "----" not in line or not first: - if last_line: - new_lines += [last_line] - # When we get to the first header ----- line, that means we need to - # add the new "minor release" section (at the top). - # If we end up auto-generating the entire index.rst, this will change - # (since we'll just loop through all the release notes, and write out - # the sections one by one. We wouldn't have to keep track of where we - # were at all). - if "----" in line and first: - if add_version == "latest": - first = False - new_lines += ["Latest (version on cutting edge of git)\n"] - new_lines += ["---------------------------------------\n"] - new_lines += ["\n"] - new_lines += [".. toctree::\n"] - new_lines += [" :maxdepth: 1\n"] - new_lines += ["\n"] - new_lines += [" latest\n"] - new_lines += ["\n"] - new_lines += [last_line] - # Something like this to add actual version to index.rst - # elif last_line != f"Version {minor_version}": - # new_lines += ["Version {minor_version}"] - # new_lines += ["-----------------------"] - # new_linst += [""] - last_line = line - - new_lines += [last_line] -# Now write all the lines back out, with the new "latest" section inserted. +from sys import argv +from glob import glob +from pathlib import Path +from os.path import basename, exists + +index_filename = Path(argv[1]) +release_note_path = Path(argv[2]) + +# Note this does NOT include the high level version summaries - those live +# in docs/source/releases/version_summaries.yaml, but are not currently being +# populated in index.rst. When this release note index creation is moved to +# brassy, the version summaries will be pulled from version_summaries.yaml +# to include in the index.rst. + +# Note this will sort release notes properly named as: +# * vX_Y_Z.rst +# * vX_Y_ZaN.rst +# * X.Y.Z.rst +# * X.Y.ZaN.rst + +# Includes "latest" at the top if it exists. + +# No other release note name formatting supported. + +# Include standard top level headers +lines = [".. dropdown:: Distribution Statement\n"] +lines += ["\n"] +lines += [" | This is an auto-generated file.\n"] +lines += [" | Please abide by license packaged with this software.\n"] +lines += ["\n"] +lines += [".. _release_notes:\n"] +lines += ["\n"] +lines += ["Release Notes\n"] +lines += ["*************\n"] +lines += ["\n"] + +# If latest.rst exists, include it in the index.rst +if exists(str(release_note_path / "latest.rst")): + lines += ["Latest (version on cutting edge of git)\n"] + lines += ["---------------------------------------\n"] + lines += ["\n"] + lines += [".. toctree::\n"] + lines += [" :maxdepth: 1\n"] + lines += ["\n"] + lines += [" latest\n"] + lines += ["\n"] + +# Collect all the major, minor, bugfix, alpha version numbers, in a sortable +# manner +versions = {} +# List all the release notes of format v*.rst - ie, don't include index.rst +# or latest.rst, but DO include X_Y_ZaN.rst (alpha releases) +for rst_file in glob(str(release_note_path / "*.rst")): + if basename(rst_file) in ["latest.rst", "index.rst"]: + continue + # print(f"Adding {rst_file}") + # Just leave X.Y.ZaN or X.Y.Z, as appropriate + # This will work with vX_Y_Z.rst, vX_Y_ZaN.rst, X.Y.Z.rst, or X.Y.ZaN.rst + version_num = ( + basename(rst_file).replace("v", "").replace(".rst", "").replace("_", ".") + ) + # print(f"verson {version_num}") + # Default alpha to None and mmb (major/minor/bugfix) to version_num - + # alpha not defined for full releases. + alpha = None + mmb_version_num = version_num + # If there is an "a" in version_num, ie X_Y_ZaN, ensure we capture the + # appropriate alpha version, and remove the aN from the mmb_version_num + if "a" in version_num: + mmb_version_num, alpha = version_num.split("a") + # Now split for major/minor/bugfix from the X_Y_Z version + major, minor, bugfix = mmb_version_num.split(".") + # Zero pad these for sorting + major = f"{int(major):03}" + minor = f"{int(minor):03}" + bugfix = f"{int(bugfix):03}" + + # Only include alpha if it is defined. + if alpha is not None: + alpha = f"{int(alpha):03}" + # Store these in dictionaries so we can loop through in order + if major not in versions: + versions[major] = {} + if minor not in versions[major]: + versions[major][minor] = {} + if bugfix not in versions[major][minor]: + versions[major][minor][bugfix] = [] + # If alphas are defined, store them in a list at the lowest level. + if alpha is not None: + versions[major][minor][bugfix] += [alpha] + +# Now loop through major, minor, bugfix, and alpha, appending sections +# in order as appropriate. +for major in sorted(versions, reverse=True): + for minor in sorted(versions[major], reverse=True): + print(f"Adding {int(major)}.{int(minor)}") + version_header = f"Version {int(major)}.{int(minor)}" + version_underline = "-" * (len(version_header)) + lines += [f"{version_header}\n"] + lines += [f"{version_underline}\n"] + lines += ["\n"] + lines += [".. toctree::\n"] + lines += [" :maxdepth: 1\n"] + lines += ["\n"] + for bugfix in sorted(versions[major][minor], reverse=True): + curr_mmb = f"v{int(major)}_{int(minor)}_{int(bugfix)}" + curr_mmb_period = f"{int(major)}.{int(minor)}.{int(bugfix)}" + # Only add major/minor/bugfix version if it exists - could be + # an alpha release with NO MMB release. + if exists(str(release_note_path / f"{curr_mmb}.rst")): + lines += [f" {curr_mmb}\n"] + elif exists(str(release_note_path / f"{curr_mmb_period}.rst")): + lines += [f" {curr_mmb_period}\n"] + for alpha in sorted(versions[major][minor][bugfix], reverse=True): + if exists(str(release_note_path / f"{curr_mmb}a{int(alpha)}.rst")): + lines += [f" {curr_mmb}a{int(alpha)}\n"] + elif exists( + str(release_note_path / f"{curr_mmb_period}a{int(alpha)}.rst") + ): + lines += [f" {curr_mmb_period}a{int(alpha)}\n"] + lines += ["\n"] +# Now write all the lines we've collected out to index.rst with open(index_filename, "w") as fobj: - fobj.writelines(new_lines) + print(f"Writing {index_filename}") + fobj.writelines(lines) diff --git a/environments/mamba_base_package_list_1.13.1_20240618.yml b/environments/mamba_base_package_list_1.13.1_20240618.yml new file mode 100644 index 000000000..04b66e1c5 --- /dev/null +++ b/environments/mamba_base_package_list_1.13.1_20240618.yml @@ -0,0 +1,211 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +_libgcc_mutex=0.1=conda_forge +_openmp_mutex=4.5=2_gnu +accessible-pygments=0.0.5=pypi_0 +affine=2.4.0=pypi_0 +alabaster=0.7.16=pypi_0 +asciitree=0.3.3=pypi_0 +astroid=3.2.2=pypi_0 +astropy=6.1.1=pypi_0 +astropy-iers-data=0.2024.6.17.0.31.35=pypi_0 +asttokens=2.4.1=pypi_0 +attrs=23.2.0=pypi_0 +babel=2.15.0=pypi_0 +bandit=1.7.9=pypi_0 +beautifulsoup4=4.12.3=pypi_0 +binutils_impl_linux-64=2.40=ha1999f0_7 +black=24.4.2=pypi_0 +bzip2=1.0.8=hd590300_5 +c-ares=1.28.1=hd590300_0 +ca-certificates=2024.6.2=hbcca054_0 +cartopy=0.23.0=pypi_0 +certifi=2024.6.2=pypi_0 +cftime=1.6.4=pypi_0 +charset-normalizer=3.3.2=pypi_0 +click=8.1.7=pypi_0 +click-plugins=1.1.1=pypi_0 +cligj=0.7.2=pypi_0 +cloudpickle=3.0.0=pypi_0 +colorama=0.4.6=pypi_0 +configobj=5.0.8=pypi_0 +contourpy=1.2.1=pypi_0 +coverage=7.5.3=pypi_0 +curl=8.8.0=he654da7_0 +cycler=0.12.1=pypi_0 +dask=2024.6.0=pypi_0 +decorator=5.1.1=pypi_0 +dill=0.3.8=pypi_0 +doc8=1.1.1=pypi_0 +docutils=0.20.1=pypi_0 +donfig=0.8.1.post1=pypi_0 +ephem=4.1.5=pypi_0 +exceptiongroup=1.2.1=pypi_0 +executing=2.0.1=pypi_0 +fasteners=0.19=pypi_0 +flake8=7.1.0=pypi_0 +flake8-docstrings=1.7.0=pypi_0 +flake8-rst=0.8.0=pypi_0 +flake8-rst-docstrings=0.3.0=pypi_0 +fonttools=4.53.0=pypi_0 +fsspec=2024.6.0=pypi_0 +gcc=13.2.0=hc7bed06_10 +gcc_impl_linux-64=13.2.0=h9eb54c0_10 +geoips=1.13.0.post391.dev0=pypi_0 +gettext=0.22.5=h59595ed_2 +gettext-tools=0.22.5=h59595ed_2 +git=2.44.0=pl5321h709897a_0 +git-lfs=3.3.0=ha770c72_0 +gxx=13.2.0=hc7bed06_10 +gxx_impl_linux-64=13.2.0=h2a599c4_10 +h5py=3.11.0=pypi_0 +idna=3.7=pypi_0 +imageio=2.34.1=pypi_0 +imagesize=1.4.1=pypi_0 +importlib-metadata=7.1.0=pypi_0 +iniconfig=2.0.0=pypi_0 +ipython=8.25.0=pypi_0 +isodate=0.6.1=pypi_0 +isort=5.13.2=pypi_0 +jedi=0.19.1=pypi_0 +jinja2=3.1.4=pypi_0 +jsonschema=4.22.0=pypi_0 +jsonschema-specifications=2023.12.1=pypi_0 +kernel-headers_linux-64=2.6.32=he073ed8_17 +keyutils=1.6.1=h166bdaf_0 +kiwisolver=1.4.5=pypi_0 +krb5=1.21.2=h659d440_0 +lazy-loader=0.4=pypi_0 +ld_impl_linux-64=2.40=hf3520f5_7 +libasprintf=0.22.5=h661eb56_2 +libasprintf-devel=0.22.5=h661eb56_2 +libcurl=8.8.0=hca28451_0 +libedit=3.1.20191231=he28a2e2_2 +libev=4.33=hd590300_2 +libexpat=2.6.2=h59595ed_0 +libffi=3.4.2=h7f98852_5 +libgcc-devel_linux-64=13.2.0=hdb50d1a_110 +libgcc-ng=13.2.0=h77fa898_10 +libgettextpo=0.22.5=h59595ed_2 +libgettextpo-devel=0.22.5=h59595ed_2 +libgfortran-ng=13.2.0=h69a702a_10 +libgfortran5=13.2.0=h3d2ce59_10 +libgomp=13.2.0=h77fa898_10 +libiconv=1.17=hd590300_2 +libnghttp2=1.58.0=h47da74e_1 +libnsl=2.0.1=hd590300_0 +libopenblas=0.3.27=pthreads_h413a1c8_0 +libsanitizer=13.2.0=h6ddb7a1_10 +libsqlite=3.46.0=hde9e2c9_0 +libssh2=1.11.0=h0841786_0 +libstdcxx-devel_linux-64=13.2.0=hdb50d1a_110 +libstdcxx-ng=13.2.0=hc0a3c3a_10 +libuuid=2.38.1=h0b41bf4_0 +libxcrypt=4.4.36=hd590300_1 +libzlib=1.3.1=h4ab18f5_1 +locket=1.0.0=pypi_0 +m2r2=0.3.3.post2=pypi_0 +markdown-it-py=3.0.0=pypi_0 +markupsafe=2.1.5=pypi_0 +matplotlib=3.9.0=pypi_0 +matplotlib-inline=0.1.7=pypi_0 +mccabe=0.7.0=pypi_0 +mdurl=0.1.2=pypi_0 +mistune=0.8.4=pypi_0 +mypy-extensions=1.0.0=pypi_0 +ncurses=6.5=h59595ed_0 +netcdf4=1.7.1=pypi_0 +networkx=3.3=pypi_0 +numcodecs=0.12.1=pypi_0 +numexpr=2.8.4=pypi_0 +numpy=1.26.4=pypi_0 +openblas=0.3.27=pthreads_h7a3da1a_0 +openssl=3.3.1=h4ab18f5_0 +packaging=24.1=pypi_0 +pandas=2.2.2=pypi_0 +parso=0.8.4=pypi_0 +partd=1.4.2=pypi_0 +pathspec=0.12.1=pypi_0 +pbr=6.0.0=pypi_0 +pcre2=10.43=hcad00b1_0 +perl=5.32.1=7_hd590300_perl5 +pillow=10.3.0=pypi_0 +pip=24.0=pyhd8ed1ab_0 +pixelmatch=0.3.0=pypi_0 +pluggy=1.5.0=pypi_0 +pooch=1.8.2=pypi_0 +prettier=0.0.7=pypi_0 +prompt-toolkit=3.0.47=pypi_0 +psutil=5.9.8=pypi_0 +pure-eval=0.2.2=pypi_0 +pyaml-env=1.2.1=pypi_0 +pycodestyle=2.12.0=pypi_0 +pydata-sphinx-theme=0.15.3=pypi_0 +pydocstyle=6.3.0=pypi_0 +pyerfa=2.0.1.4=pypi_0 +pyflakes=3.2.0=pypi_0 +pygments=2.18.0=pypi_0 +pyhdf=0.11.4=pypi_0 +pykdtree=1.3.12=pypi_0 +pylint=3.2.3=pypi_0 +pyorbital=1.8.2=pypi_0 +pyparsing=3.1.2=pypi_0 +pyproj=3.6.1=pypi_0 +pypublicdecompwt=2.8.1.5=pypi_0 +pyresample=1.28.3=pypi_0 +pyshp=2.3.1=pypi_0 +pytest=8.2.2=pypi_0 +pytest-cov=5.0.0=pypi_0 +python=3.10.14=hd12c33a_0_cpython +python-dateutil=2.9.0.post0=pypi_0 +pytz=2024.1=pypi_0 +pyyaml=6.0.1=pypi_0 +rasterio=1.3.10=pypi_0 +readline=8.2=h8228510_1 +referencing=0.35.1=pypi_0 +requests=2.32.3=pypi_0 +restructuredtext-lint=1.4.0=pypi_0 +rich=13.7.1=pypi_0 +rpds-py=0.18.1=pypi_0 +satpy=0.49.0=pypi_0 +scikit-image=0.23.2=pypi_0 +scipy=1.13.1=pypi_0 +setuptools=70.0.0=pyhd8ed1ab_0 +shapely=2.0.4=pypi_0 +six=1.16.0=pypi_0 +snowballstemmer=2.2.0=pypi_0 +snuggs=1.4.7=pypi_0 +soupsieve=2.5=pypi_0 +sphinx=7.1.2=pypi_0 +sphinx-design=0.6.0=pypi_0 +sphinxcontrib-applehelp=1.0.8=pypi_0 +sphinxcontrib-autoprogram=0.1.9=pypi_0 +sphinxcontrib-devhelp=1.0.6=pypi_0 +sphinxcontrib-htmlhelp=2.0.5=pypi_0 +sphinxcontrib-jsmath=1.0.1=pypi_0 +sphinxcontrib-qthelp=1.0.7=pypi_0 +sphinxcontrib-serializinghtml=1.1.10=pypi_0 +stack-data=0.6.3=pypi_0 +stevedore=5.2.0=pypi_0 +sysroot_linux-64=2.12=he073ed8_17 +tabulate=0.9.0=pypi_0 +tifffile=2024.5.22=pypi_0 +tk=8.6.13=noxft_h4845f30_101 +tomli=2.0.1=pypi_0 +tomlkit=0.12.5=pypi_0 +toolz=0.12.1=pypi_0 +traitlets=5.14.3=pypi_0 +trollimage=1.23.2=pypi_0 +trollsift=0.5.1=pypi_0 +typing-extensions=4.12.2=pypi_0 +tzdata=2024.1=pypi_0 +urllib3=2.2.2=pypi_0 +wcwidth=0.2.13=pypi_0 +wheel=0.43.0=pyhd8ed1ab_1 +xarray=2024.2.0=pypi_0 +xarray-datatree=0.0.14=pypi_0 +xz=5.2.6=h166bdaf_0 +zarr=2.18.2=pypi_0 +zipp=3.19.2=pypi_0 +zstd=1.5.6=ha6fb4c9_0 diff --git a/environments/mamba_base_package_list_1.13.1_20240704.yml b/environments/mamba_base_package_list_1.13.1_20240704.yml new file mode 100644 index 000000000..c09f93fb9 --- /dev/null +++ b/environments/mamba_base_package_list_1.13.1_20240704.yml @@ -0,0 +1,218 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +_libgcc_mutex=0.1=conda_forge +_openmp_mutex=4.5=2_gnu +accessible-pygments=0.0.5=pypi_0 +affine=2.4.0=pypi_0 +alabaster=0.7.16=pypi_0 +alphashape=1.3.1=pypi_0 +asciitree=0.3.3=pypi_0 +astroid=3.2.2=pypi_0 +astropy=6.1.1=pypi_0 +astropy-iers-data=0.2024.7.1.0.34.3=pypi_0 +asttokens=2.4.1=pypi_0 +attrs=23.2.0=pypi_0 +babel=2.15.0=pypi_0 +bandit=1.7.9=pypi_0 +beautifulsoup4=4.12.3=pypi_0 +binutils_impl_linux-64=2.40=ha1999f0_7 +black=24.4.2=pypi_0 +bzip2=1.0.8=hd590300_5 +c-ares=1.28.1=hd590300_0 +ca-certificates=2024.7.4=hbcca054_0 +cartopy=0.23.0=pypi_0 +certifi=2024.7.4=pypi_0 +cftime=1.6.4=pypi_0 +charset-normalizer=3.3.2=pypi_0 +click=8.1.7=pypi_0 +click-log=0.4.0=pypi_0 +click-plugins=1.1.1=pypi_0 +cligj=0.7.2=pypi_0 +cloudpickle=3.0.0=pypi_0 +colorama=0.4.6=pypi_0 +configobj=5.0.8=pypi_0 +contourpy=1.2.1=pypi_0 +coverage=7.5.4=pypi_0 +curl=8.8.0=he654da7_1 +cycler=0.12.1=pypi_0 +dask=2024.6.2=pypi_0 +decorator=5.1.1=pypi_0 +dill=0.3.8=pypi_0 +doc8=1.1.1=pypi_0 +docutils=0.20.1=pypi_0 +donfig=0.8.1.post1=pypi_0 +ephem=4.1.5=pypi_0 +exceptiongroup=1.2.1=pypi_0 +executing=2.0.1=pypi_0 +fasteners=0.19=pypi_0 +flake8=7.1.0=pypi_0 +flake8-docstrings=1.7.0=pypi_0 +flake8-rst=0.8.0=pypi_0 +flake8-rst-docstrings=0.3.0=pypi_0 +fonttools=4.53.0=pypi_0 +fsspec=2024.6.1=pypi_0 +gcc=14.1.0=h6f9ffa1_0 +gcc_impl_linux-64=14.1.0=h3c94d91_0 +geoips=1.13.1.post1.dev0=pypi_0 +gettext=0.22.5=h59595ed_2 +gettext-tools=0.22.5=h59595ed_2 +git=2.44.0=pl5321h709897a_0 +git-lfs=3.3.0=ha770c72_0 +gxx=14.1.0=h6f9ffa1_0 +gxx_impl_linux-64=14.1.0=h2879b86_0 +h5py=3.11.0=pypi_0 +idna=3.7=pypi_0 +imageio=2.34.2=pypi_0 +imagesize=1.4.1=pypi_0 +importlib-metadata=8.0.0=pypi_0 +iniconfig=2.0.0=pypi_0 +ipython=8.26.0=pypi_0 +isodate=0.6.1=pypi_0 +isort=5.13.2=pypi_0 +jedi=0.19.1=pypi_0 +jinja2=3.1.4=pypi_0 +jsonschema=4.22.0=pypi_0 +jsonschema-specifications=2023.12.1=pypi_0 +kernel-headers_linux-64=2.6.32=he073ed8_17 +keyutils=1.6.1=h166bdaf_0 +kiwisolver=1.4.5=pypi_0 +krb5=1.21.3=h659f571_0 +lazy-loader=0.4=pypi_0 +ld_impl_linux-64=2.40=hf3520f5_7 +libasprintf=0.22.5=h661eb56_2 +libasprintf-devel=0.22.5=h661eb56_2 +libcurl=8.8.0=hca28451_1 +libedit=3.1.20191231=he28a2e2_2 +libev=4.33=hd590300_2 +libexpat=2.6.2=h59595ed_0 +libffi=3.4.2=h7f98852_5 +libgcc-devel_linux-64=14.1.0=h5d3d1c9_100 +libgcc-ng=14.1.0=h77fa898_0 +libgettextpo=0.22.5=h59595ed_2 +libgettextpo-devel=0.22.5=h59595ed_2 +libgfortran-ng=14.1.0=h69a702a_0 +libgfortran5=14.1.0=hc5f4f2c_0 +libgomp=14.1.0=h77fa898_0 +libiconv=1.17=hd590300_2 +libnghttp2=1.58.0=h47da74e_1 +libnsl=2.0.1=hd590300_0 +libopenblas=0.3.27=pthreads_h413a1c8_0 +libsanitizer=14.1.0=hcba0ae0_0 +libsqlite=3.46.0=hde9e2c9_0 +libssh2=1.11.0=h0841786_0 +libstdcxx-devel_linux-64=14.1.0=h5d3d1c9_100 +libstdcxx-ng=14.1.0=hc0a3c3a_0 +libuuid=2.38.1=h0b41bf4_0 +libxcrypt=4.4.36=hd590300_1 +libzlib=1.3.1=h4ab18f5_1 +locket=1.0.0=pypi_0 +m2r2=0.3.3.post2=pypi_0 +markdown-it-py=3.0.0=pypi_0 +markupsafe=2.1.5=pypi_0 +matplotlib=3.9.1=pypi_0 +matplotlib-inline=0.1.7=pypi_0 +mccabe=0.7.0=pypi_0 +mdurl=0.1.2=pypi_0 +mistune=0.8.4=pypi_0 +mypy-extensions=1.0.0=pypi_0 +ncurses=6.5=h59595ed_0 +netcdf4=1.6.5=pypi_0 +networkx=3.3=pypi_0 +numcodecs=0.12.1=pypi_0 +numexpr=2.8.4=pypi_0 +numpy=1.26.4=pypi_0 +openblas=0.3.27=pthreads_h7a3da1a_0 +openssl=3.3.1=h4ab18f5_1 +packaging=24.1=pypi_0 +pandas=2.2.2=pypi_0 +parso=0.8.4=pypi_0 +partd=1.4.2=pypi_0 +pathspec=0.12.1=pypi_0 +pbr=6.0.0=pypi_0 +pcre2=10.43=hcad00b1_0 +perl=5.32.1=7_hd590300_perl5 +pexpect=4.9.0=pypi_0 +pillow=10.4.0=pypi_0 +pip=24.0=pyhd8ed1ab_0 +pixelmatch=0.3.0=pypi_0 +platformdirs=4.2.2=pypi_0 +pluggy=1.5.0=pypi_0 +pooch=1.8.2=pypi_0 +prettier=0.0.7=pypi_0 +prompt-toolkit=3.0.47=pypi_0 +psutil=6.0.0=pypi_0 +ptyprocess=0.7.0=pypi_0 +pure-eval=0.2.2=pypi_0 +pyaml-env=1.2.1=pypi_0 +pycodestyle=2.12.0=pypi_0 +pydata-sphinx-theme=0.15.4=pypi_0 +pydocstyle=6.3.0=pypi_0 +pyerfa=2.0.1.4=pypi_0 +pyflakes=3.2.0=pypi_0 +pygments=2.18.0=pypi_0 +pyhdf=0.11.4=pypi_0 +pykdtree=1.3.12=pypi_0 +pylint=3.2.5=pypi_0 +pyorbital=1.8.3=pypi_0 +pyparsing=3.1.2=pypi_0 +pyproj=3.6.1=pypi_0 +pypublicdecompwt=2.8.1.5=pypi_0 +pyresample=1.28.4=pypi_0 +pyshp=2.3.1=pypi_0 +pytest=8.2.2=pypi_0 +pytest-cov=5.0.0=pypi_0 +python=3.10.14=hd12c33a_0_cpython +python-dateutil=2.9.0.post0=pypi_0 +pytz=2024.1=pypi_0 +pyyaml=6.0.1=pypi_0 +rasterio=1.3.10=pypi_0 +readline=8.2=h8228510_1 +referencing=0.35.1=pypi_0 +requests=2.32.3=pypi_0 +restructuredtext-lint=1.4.0=pypi_0 +rich=13.7.1=pypi_0 +rpds-py=0.18.1=pypi_0 +rtree=1.2.0=pypi_0 +satpy=0.49.0=pypi_0 +scikit-image=0.24.0=pypi_0 +scipy=1.14.0=pypi_0 +setuptools=70.1.1=pyhd8ed1ab_0 +shapely=2.0.4=pypi_0 +six=1.16.0=pypi_0 +snowballstemmer=2.2.0=pypi_0 +snuggs=1.4.7=pypi_0 +soupsieve=2.5=pypi_0 +sphinx=7.1.2=pypi_0 +sphinx-design=0.6.0=pypi_0 +sphinxcontrib-applehelp=1.0.8=pypi_0 +sphinxcontrib-autoprogram=0.1.9=pypi_0 +sphinxcontrib-devhelp=1.0.6=pypi_0 +sphinxcontrib-htmlhelp=2.0.5=pypi_0 +sphinxcontrib-jsmath=1.0.1=pypi_0 +sphinxcontrib-qthelp=1.0.7=pypi_0 +sphinxcontrib-serializinghtml=1.1.10=pypi_0 +stack-data=0.6.3=pypi_0 +stevedore=5.2.0=pypi_0 +sysroot_linux-64=2.12=he073ed8_17 +tabulate=0.9.0=pypi_0 +tifffile=2024.7.2=pypi_0 +tk=8.6.13=noxft_h4845f30_101 +tomli=2.0.1=pypi_0 +tomlkit=0.12.5=pypi_0 +toolz=0.12.1=pypi_0 +traitlets=5.14.3=pypi_0 +trimesh=4.4.1=pypi_0 +trollimage=1.24.0=pypi_0 +trollsift=0.5.1=pypi_0 +typing-extensions=4.12.2=pypi_0 +tzdata=2024.1=pypi_0 +urllib3=2.2.2=pypi_0 +wcwidth=0.2.13=pypi_0 +wheel=0.43.0=pyhd8ed1ab_1 +xarray=2024.2.0=pypi_0 +xarray-datatree=0.0.14=pypi_0 +xz=5.2.6=h166bdaf_0 +zarr=2.18.2=pypi_0 +zipp=3.19.2=pypi_0 +zstd=1.5.6=ha6fb4c9_0 diff --git a/environments/mamba_base_package_list_1.14.0_20240915.yml b/environments/mamba_base_package_list_1.14.0_20240915.yml new file mode 100644 index 000000000..d04903d48 --- /dev/null +++ b/environments/mamba_base_package_list_1.14.0_20240915.yml @@ -0,0 +1,232 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +_libgcc_mutex=0.1=conda_forge +_openmp_mutex=4.5=2_gnu +_sysroot_linux-64_curr_repodata_hack=3=h69a702a_16 +accessible-pygments=0.0.5=pypi_0 +affine=2.4.0=pypi_0 +alabaster=0.7.16=pypi_0 +alphashape=1.3.1=pypi_0 +annotated-types=0.7.0=pypi_0 +asciitree=0.3.3=pypi_0 +astroid=3.2.4=pypi_0 +astropy=6.1.3=pypi_0 +astropy-iers-data=0.2024.9.12.13.29.57=pypi_0 +asttokens=2.4.1=pypi_0 +attrs=24.2.0=pypi_0 +babel=2.16.0=pypi_0 +bandit=1.7.9=pypi_0 +beautifulsoup4=4.12.3=pypi_0 +binutils_impl_linux-64=2.40=ha1999f0_7 +black=24.8.0=pypi_0 +brassy=0.0.2.post2=pypi_0 +bzip2=1.0.8=h4bc722e_7 +c-ares=1.32.3=h4bc722e_0 +ca-certificates=2024.8.30=hbcca054_0 +cartopy=0.23.0=pypi_0 +certifi=2024.8.30=pypi_0 +cffi=1.17.1=pypi_0 +cftime=1.6.4=pypi_0 +charset-normalizer=3.3.2=pypi_0 +click=8.1.7=pypi_0 +click-log=0.4.0=pypi_0 +click-plugins=1.1.1=pypi_0 +cligj=0.7.2=pypi_0 +cloudpickle=3.0.0=pypi_0 +colorama=0.4.6=pypi_0 +configobj=5.0.8=pypi_0 +contourpy=1.3.0=pypi_0 +coverage=7.6.1=pypi_0 +curl=8.10.0=hbbe4b11_0 +cycler=0.12.1=pypi_0 +dask=2024.9.0=pypi_0 +decorator=5.1.1=pypi_0 +dill=0.3.8=pypi_0 +doc8=1.1.2=pypi_0 +docutils=0.20.1=pypi_0 +donfig=0.8.1.post1=pypi_0 +ephem=4.1.5=pypi_0 +exceptiongroup=1.2.2=pypi_0 +executing=2.1.0=pypi_0 +fasteners=0.19=pypi_0 +flake8=7.1.1=pypi_0 +flake8-docstrings=1.7.0=pypi_0 +flake8-rst=0.8.0=pypi_0 +flake8-rst-docstrings=0.3.0=pypi_0 +fonttools=4.53.1=pypi_0 +fsspec=2024.9.0=pypi_0 +gcc=14.1.0=h6f9ffa1_1 +gcc_impl_linux-64=14.1.0=h3c94d91_1 +geoips=1.14.0=pypi_0 +gettext=0.22.5=he02047a_3 +gettext-tools=0.22.5=he02047a_3 +git=2.44.0=pl5321h709897a_0 +git-lfs=3.3.0=ha770c72_0 +gxx=14.1.0=h6f9ffa1_1 +gxx_impl_linux-64=14.1.0=h8d00ecb_1 +h5py=3.11.0=pypi_0 +idna=3.9=pypi_0 +imageio=2.35.1=pypi_0 +imagesize=1.4.1=pypi_0 +importlib-metadata=8.5.0=pypi_0 +iniconfig=2.0.0=pypi_0 +ipython=8.27.0=pypi_0 +isodate=0.6.1=pypi_0 +isort=5.13.2=pypi_0 +jedi=0.19.1=pypi_0 +jinja2=3.1.4=pypi_0 +jsonschema=4.23.0=pypi_0 +jsonschema-specifications=2023.12.1=pypi_0 +kernel-headers_linux-64=3.10.0=h4a8ded7_16 +keyutils=1.6.1=h166bdaf_0 +kiwisolver=1.4.7=pypi_0 +krb5=1.21.3=h659f571_0 +lazy-loader=0.4=pypi_0 +ld_impl_linux-64=2.40=hf3520f5_7 +libasprintf=0.22.5=he8f35ee_3 +libasprintf-devel=0.22.5=he8f35ee_3 +libcurl=8.10.0=hbbe4b11_0 +libedit=3.1.20191231=he28a2e2_2 +libev=4.33=hd590300_2 +libexpat=2.6.3=h5888daf_0 +libffi=3.4.2=h7f98852_5 +libgcc=14.1.0=h77fa898_1 +libgcc-devel_linux-64=14.1.0=h5d3d1c9_101 +libgcc-ng=14.1.0=h69a702a_1 +libgettextpo=0.22.5=he02047a_3 +libgettextpo-devel=0.22.5=he02047a_3 +libgfortran=14.1.0=h69a702a_1 +libgfortran-ng=14.1.0=h69a702a_1 +libgfortran5=14.1.0=hc5f4f2c_1 +libgomp=14.1.0=h77fa898_1 +libiconv=1.17=hd590300_2 +libnghttp2=1.58.0=h47da74e_1 +libnsl=2.0.1=hd590300_0 +libopenblas=0.3.28=pthreads_h94d23a6_0 +libsanitizer=14.1.0=hcba0ae0_1 +libsqlite=3.46.1=hadc24fc_0 +libssh2=1.11.0=h0841786_0 +libstdcxx=14.1.0=hc0a3c3a_1 +libstdcxx-devel_linux-64=14.1.0=h5d3d1c9_101 +libstdcxx-ng=14.1.0=h4852527_1 +libuuid=2.38.1=h0b41bf4_0 +libxcrypt=4.4.36=hd590300_1 +libzlib=1.3.1=h4ab18f5_1 +locket=1.0.0=pypi_0 +m2r2=0.3.3.post2=pypi_0 +markdown-it-py=3.0.0=pypi_0 +markupsafe=2.1.5=pypi_0 +matplotlib=3.9.2=pypi_0 +matplotlib-inline=0.1.7=pypi_0 +mccabe=0.7.0=pypi_0 +mdurl=0.1.2=pypi_0 +mistune=0.8.4=pypi_0 +morecantile=5.4.2=pypi_0 +mypy-extensions=1.0.0=pypi_0 +ncurses=6.5=he02047a_1 +netcdf4=1.7.1.post2=pypi_0 +networkx=3.3=pypi_0 +numcodecs=0.13.0=pypi_0 +numexpr=2.8.4=pypi_0 +numpy=1.26.4=pypi_0 +openblas=0.3.28=pthreads_hbcdf1e8_0 +openssl=3.3.2=hb9d3cd8_0 +packaging=24.1=pypi_0 +pandas=2.2.2=pypi_0 +parso=0.8.4=pypi_0 +partd=1.4.2=pypi_0 +pathspec=0.12.1=pypi_0 +pbr=6.1.0=pypi_0 +pcre2=10.43=hcad00b1_0 +perl=5.32.1=7_hd590300_perl5 +pillow=10.4.0=pypi_0 +pinkrst=0.0.1=pypi_0 +pip=24.2=pyh8b19718_1 +pixelmatch=0.3.0=pypi_0 +platformdirs=4.3.3=pypi_0 +pluggy=1.5.0=pypi_0 +pooch=1.8.2=pypi_0 +prettier=0.0.7=pypi_0 +prompt-toolkit=3.0.47=pypi_0 +psutil=6.0.0=pypi_0 +pure-eval=0.2.3=pypi_0 +pyaml-env=1.2.1=pypi_0 +pycodestyle=2.12.1=pypi_0 +pycparser=2.22=pypi_0 +pydantic=2.9.1=pypi_0 +pydantic-core=2.23.3=pypi_0 +pydata-sphinx-theme=0.15.4=pypi_0 +pydocstyle=6.3.0=pypi_0 +pyerfa=2.0.1.4=pypi_0 +pyflakes=3.2.0=pypi_0 +pygit2=1.15.1=pypi_0 +pygments=2.18.0=pypi_0 +pyhdf=0.11.4=pypi_0 +pykdtree=1.3.13=pypi_0 +pylint=3.2.7=pypi_0 +pyorbital=1.8.3=pypi_0 +pyparsing=3.1.4=pypi_0 +pyproj=3.6.1=pypi_0 +pypublicdecompwt=2.8.1.6=pypi_0 +pyresample=1.30.0=pypi_0 +pyshp=2.3.1=pypi_0 +pytest=8.3.3=pypi_0 +pytest-cov=5.0.0=pypi_0 +python=3.10.14=hd12c33a_0_cpython +python-dateutil=2.9.0.post0=pypi_0 +pytz=2024.2=pypi_0 +pyyaml=6.0.2=pypi_0 +rasterio=1.3.11=pypi_0 +readline=8.2=h8228510_1 +referencing=0.35.1=pypi_0 +requests=2.32.3=pypi_0 +restructuredtext-lint=1.4.0=pypi_0 +rich=13.8.1=pypi_0 +rich-argparse=1.5.2=pypi_0 +rio-cogeo=5.3.4=pypi_0 +rpds-py=0.20.0=pypi_0 +rtree=1.3.0=pypi_0 +satpy=0.51.0=pypi_0 +scikit-image=0.24.0=pypi_0 +scipy=1.14.1=pypi_0 +setuptools=73.0.1=pyhd8ed1ab_0 +shapely=2.0.6=pypi_0 +six=1.16.0=pypi_0 +snowballstemmer=2.2.0=pypi_0 +snuggs=1.4.7=pypi_0 +soupsieve=2.6=pypi_0 +sphinx=7.1.2=pypi_0 +sphinx-argparse=0.5.2=pypi_0 +sphinx-design=0.6.1=pypi_0 +sphinxcontrib-applehelp=2.0.0=pypi_0 +sphinxcontrib-autoprogram=0.1.9=pypi_0 +sphinxcontrib-devhelp=2.0.0=pypi_0 +sphinxcontrib-htmlhelp=2.1.0=pypi_0 +sphinxcontrib-jsmath=1.0.1=pypi_0 +sphinxcontrib-qthelp=2.0.0=pypi_0 +sphinxcontrib-serializinghtml=2.0.0=pypi_0 +stack-data=0.6.3=pypi_0 +stevedore=5.3.0=pypi_0 +sysroot_linux-64=2.17=h4a8ded7_16 +tabulate=0.9.0=pypi_0 +tifffile=2024.8.30=pypi_0 +tk=8.6.13=noxft_h4845f30_101 +tomli=2.0.1=pypi_0 +tomlkit=0.13.2=pypi_0 +toolz=0.12.1=pypi_0 +traitlets=5.14.3=pypi_0 +trimesh=4.4.9=pypi_0 +trollimage=1.25.0=pypi_0 +trollsift=0.5.1=pypi_0 +typing-extensions=4.12.2=pypi_0 +tzdata=2024.1=pypi_0 +urllib3=2.2.3=pypi_0 +wcwidth=0.2.13=pypi_0 +wheel=0.44.0=pyhd8ed1ab_0 +xarray=2024.2.0=pypi_0 +xarray-datatree=0.0.14=pypi_0 +xz=5.2.6=h166bdaf_0 +zarr=2.18.3=pypi_0 +zipp=3.20.2=pypi_0 +zstd=1.5.6=ha6fb4c9_0 diff --git a/environments/mamba_full_package_list_1.14.0_20240916.yml b/environments/mamba_full_package_list_1.14.0_20240916.yml new file mode 100644 index 000000000..e1d59e23b --- /dev/null +++ b/environments/mamba_full_package_list_1.14.0_20240916.yml @@ -0,0 +1,239 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +_libgcc_mutex=0.1=conda_forge +_openmp_mutex=4.5=2_gnu +_sysroot_linux-64_curr_repodata_hack=3=h69a702a_16 +accessible-pygments=0.0.5=pypi_0 +affine=2.4.0=pypi_0 +akima86=0.1.1=pypi_0 +alabaster=0.7.16=pypi_0 +alphashape=1.3.1=pypi_0 +annotated-types=0.7.0=pypi_0 +archer=1.0=pypi_0 +asciitree=0.3.3=pypi_0 +astroid=3.2.4=pypi_0 +astropy=6.1.3=pypi_0 +astropy-iers-data=0.2024.9.12.13.29.57=pypi_0 +asttokens=2.4.1=pypi_0 +attrs=24.2.0=pypi_0 +babel=2.16.0=pypi_0 +bandit=1.7.9=pypi_0 +beautifulsoup4=4.12.3=pypi_0 +binutils_impl_linux-64=2.40=ha1999f0_7 +black=24.8.0=pypi_0 +brassy=0.0.2.post2=pypi_0 +bzip2=1.0.8=h4bc722e_7 +c-ares=1.32.3=h4bc722e_0 +ca-certificates=2024.8.30=hbcca054_0 +cartopy=0.23.0=pypi_0 +certifi=2024.8.30=pypi_0 +cffi=1.17.1=pypi_0 +cftime=1.6.4=pypi_0 +charset-normalizer=3.3.2=pypi_0 +click=8.1.7=pypi_0 +click-log=0.4.0=pypi_0 +click-plugins=1.1.1=pypi_0 +cligj=0.7.2=pypi_0 +cloudpickle=3.0.0=pypi_0 +colorama=0.4.6=pypi_0 +configobj=5.0.8=pypi_0 +contourpy=1.3.0=pypi_0 +coverage=7.6.1=pypi_0 +curl=8.10.0=hbbe4b11_0 +cycler=0.12.1=pypi_0 +dask=2024.9.0=pypi_0 +data-fusion=1.14.0=pypi_0 +decorator=5.1.1=pypi_0 +dill=0.3.8=pypi_0 +doc8=1.1.2=pypi_0 +docutils=0.20.1=pypi_0 +donfig=0.8.1.post1=pypi_0 +ephem=4.1.5=pypi_0 +exceptiongroup=1.2.2=pypi_0 +executing=2.1.0=pypi_0 +fasteners=0.19=pypi_0 +flake8=7.1.1=pypi_0 +flake8-docstrings=1.7.0=pypi_0 +flake8-rst=0.8.0=pypi_0 +flake8-rst-docstrings=0.3.0=pypi_0 +fonttools=4.53.1=pypi_0 +fsspec=2024.9.0=pypi_0 +gcc=14.1.0=h6f9ffa1_1 +gcc_impl_linux-64=14.1.0=h3c94d91_1 +geoips=1.14.0=pypi_0 +geoips-clavrx=1.13.0.post1.dev0=pypi_0 +geoips-plugin-example=1.13.0.post1.dev0=pypi_0 +gettext=0.22.5=he02047a_3 +gettext-tools=0.22.5=he02047a_3 +git=2.44.0=pl5321h709897a_0 +git-lfs=3.3.0=ha770c72_0 +gxx=14.1.0=h6f9ffa1_1 +gxx_impl_linux-64=14.1.0=h8d00ecb_1 +h5py=3.11.0=pypi_0 +idna=3.9=pypi_0 +imageio=2.35.1=pypi_0 +imagesize=1.4.1=pypi_0 +importlib-metadata=8.5.0=pypi_0 +iniconfig=2.0.0=pypi_0 +ipython=8.27.0=pypi_0 +isodate=0.6.1=pypi_0 +isort=5.13.2=pypi_0 +jedi=0.19.1=pypi_0 +jinja2=3.1.4=pypi_0 +jsonschema=4.23.0=pypi_0 +jsonschema-specifications=2023.12.1=pypi_0 +kernel-headers_linux-64=3.10.0=h4a8ded7_16 +keyutils=1.6.1=h166bdaf_0 +kiwisolver=1.4.7=pypi_0 +krb5=1.21.3=h659f571_0 +lazy-loader=0.4=pypi_0 +ld_impl_linux-64=2.40=hf3520f5_7 +libasprintf=0.22.5=he8f35ee_3 +libasprintf-devel=0.22.5=he8f35ee_3 +libcurl=8.10.0=hbbe4b11_0 +libedit=3.1.20191231=he28a2e2_2 +libev=4.33=hd590300_2 +libexpat=2.6.3=h5888daf_0 +libffi=3.4.2=h7f98852_5 +libgcc=14.1.0=h77fa898_1 +libgcc-devel_linux-64=14.1.0=h5d3d1c9_101 +libgcc-ng=14.1.0=h69a702a_1 +libgettextpo=0.22.5=he02047a_3 +libgettextpo-devel=0.22.5=he02047a_3 +libgfortran=14.1.0=h69a702a_1 +libgfortran-ng=14.1.0=h69a702a_1 +libgfortran5=14.1.0=hc5f4f2c_1 +libgomp=14.1.0=h77fa898_1 +libiconv=1.17=hd590300_2 +libnghttp2=1.58.0=h47da74e_1 +libnsl=2.0.1=hd590300_0 +libopenblas=0.3.28=pthreads_h94d23a6_0 +libsanitizer=14.1.0=hcba0ae0_1 +libsqlite=3.46.1=hadc24fc_0 +libssh2=1.11.0=h0841786_0 +libstdcxx=14.1.0=hc0a3c3a_1 +libstdcxx-devel_linux-64=14.1.0=h5d3d1c9_101 +libstdcxx-ng=14.1.0=h4852527_1 +libuuid=2.38.1=h0b41bf4_0 +libxcrypt=4.4.36=hd590300_1 +libzlib=1.3.1=h4ab18f5_1 +locket=1.0.0=pypi_0 +m2r2=0.3.3.post2=pypi_0 +markdown-it-py=3.0.0=pypi_0 +markupsafe=2.1.5=pypi_0 +matplotlib=3.9.2=pypi_0 +matplotlib-inline=0.1.7=pypi_0 +mccabe=0.7.0=pypi_0 +mdurl=0.1.2=pypi_0 +mistune=0.8.4=pypi_0 +morecantile=5.4.2=pypi_0 +my-package=1.13.1=pypi_0 +mypy-extensions=1.0.0=pypi_0 +ncurses=6.5=he02047a_1 +netcdf4=1.7.1.post2=pypi_0 +networkx=3.3=pypi_0 +numcodecs=0.13.0=pypi_0 +numexpr=2.8.4=pypi_0 +numpy=1.26.4=pypi_0 +openblas=0.3.28=pthreads_hbcdf1e8_0 +openssl=3.3.2=hb9d3cd8_0 +packaging=24.1=pypi_0 +pandas=2.2.2=pypi_0 +parso=0.8.4=pypi_0 +partd=1.4.2=pypi_0 +pathspec=0.12.1=pypi_0 +pbr=6.1.0=pypi_0 +pcre2=10.43=hcad00b1_0 +perl=5.32.1=7_hd590300_perl5 +pillow=10.4.0=pypi_0 +pinkrst=0.0.1=pypi_0 +pip=24.2=pyh8b19718_1 +pixelmatch=0.3.0=pypi_0 +platformdirs=4.3.3=pypi_0 +pluggy=1.5.0=pypi_0 +pooch=1.8.2=pypi_0 +prettier=0.0.7=pypi_0 +prompt-toolkit=3.0.47=pypi_0 +psutil=6.0.0=pypi_0 +pure-eval=0.2.3=pypi_0 +pyaml-env=1.2.1=pypi_0 +pycodestyle=2.12.1=pypi_0 +pycparser=2.22=pypi_0 +pydantic=2.9.1=pypi_0 +pydantic-core=2.23.3=pypi_0 +pydata-sphinx-theme=0.15.4=pypi_0 +pydocstyle=6.3.0=pypi_0 +pyerfa=2.0.1.4=pypi_0 +pyflakes=3.2.0=pypi_0 +pygit2=1.15.1=pypi_0 +pygments=2.18.0=pypi_0 +pyhdf=0.11.4=pypi_0 +pykdtree=1.3.13=pypi_0 +pylint=3.2.7=pypi_0 +pyorbital=1.8.3=pypi_0 +pyparsing=3.1.4=pypi_0 +pyproj=3.6.1=pypi_0 +pypublicdecompwt=2.8.1.6=pypi_0 +pyresample=1.30.0=pypi_0 +pyshp=2.3.1=pypi_0 +pytest=8.3.3=pypi_0 +pytest-cov=5.0.0=pypi_0 +python=3.10.14=hd12c33a_0_cpython +python-dateutil=2.9.0.post0=pypi_0 +pytz=2024.2=pypi_0 +pyyaml=6.0.2=pypi_0 +rasterio=1.3.11=pypi_0 +readline=8.2=h8228510_1 +recenter-tc=1.14.0a0.post3.dev0=pypi_0 +referencing=0.35.1=pypi_0 +requests=2.32.3=pypi_0 +restructuredtext-lint=1.4.0=pypi_0 +rich=13.8.1=pypi_0 +rich-argparse=1.5.2=pypi_0 +rio-cogeo=5.3.4=pypi_0 +rpds-py=0.20.0=pypi_0 +rtree=1.3.0=pypi_0 +satpy=0.51.0=pypi_0 +scikit-image=0.24.0=pypi_0 +scipy=1.14.1=pypi_0 +setuptools=69.5.1=pypi_0 +shapely=2.0.6=pypi_0 +six=1.16.0=pypi_0 +snowballstemmer=2.2.0=pypi_0 +snuggs=1.4.7=pypi_0 +soupsieve=2.6=pypi_0 +sphinx=7.1.2=pypi_0 +sphinx-argparse=0.5.2=pypi_0 +sphinx-design=0.6.1=pypi_0 +sphinxcontrib-applehelp=2.0.0=pypi_0 +sphinxcontrib-autoprogram=0.1.9=pypi_0 +sphinxcontrib-devhelp=2.0.0=pypi_0 +sphinxcontrib-htmlhelp=2.1.0=pypi_0 +sphinxcontrib-jsmath=1.0.1=pypi_0 +sphinxcontrib-qthelp=2.0.0=pypi_0 +sphinxcontrib-serializinghtml=2.0.0=pypi_0 +stack-data=0.6.3=pypi_0 +stevedore=5.3.0=pypi_0 +sysroot_linux-64=2.17=h4a8ded7_16 +tabulate=0.9.0=pypi_0 +tifffile=2024.8.30=pypi_0 +tk=8.6.13=noxft_h4845f30_101 +tomli=2.0.1=pypi_0 +tomlkit=0.13.2=pypi_0 +toolz=0.12.1=pypi_0 +traitlets=5.14.3=pypi_0 +trimesh=4.4.9=pypi_0 +trollimage=1.25.0=pypi_0 +trollsift=0.5.1=pypi_0 +typing-extensions=4.12.2=pypi_0 +tzdata=2024.1=pypi_0 +urllib3=2.2.3=pypi_0 +wcwidth=0.2.13=pypi_0 +wheel=0.44.0=pyhd8ed1ab_0 +xarray=2024.2.0=pypi_0 +xarray-datatree=0.0.14=pypi_0 +xz=5.2.6=h166bdaf_0 +zarr=2.18.3=pypi_0 +zipp=3.20.2=pypi_0 +zstd=1.5.6=ha6fb4c9_0 diff --git a/environments/pip_base_requirements_1.13.1_20240618.txt b/environments/pip_base_requirements_1.13.1_20240618.txt new file mode 100644 index 000000000..f465a8c2f --- /dev/null +++ b/environments/pip_base_requirements_1.13.1_20240618.txt @@ -0,0 +1,175 @@ +accessible-pygments==0.0.5 +affine==2.4.0 +alabaster==0.7.16 +asciitree==0.3.3 +astroid==3.2.2 +astropy==6.1.1 +astropy-iers-data==0.2024.6.17.0.31.35 +asttokens==2.4.1 +attrs==23.2.0 +Babel==2.15.0 +bandit==1.7.9 +beautifulsoup4==4.12.3 +black==24.4.2 +CacheControl==0.13.1 +Cartopy==0.23.0 +certifi==2024.6.2 +cftime==1.6.4 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +click-plugins==1.1.1 +cligj==0.7.2 +cloudpickle==3.0.0 +colorama==0.4.6 +configobj==5.0.8 +contourpy==1.2.1 +coverage==7.5.3 +crashtest==0.4.1 +cycler==0.12.1 +dask==2024.6.0 +decorator==5.1.1 +dill==0.3.8 +distlib==0.3.8 +doc8==1.1.1 +docutils==0.20.1 +donfig==0.8.1.post1 +dulwich==0.21.7 +ephem==4.1.5 +exceptiongroup==1.2.1 +executing==2.0.1 +fasteners==0.19 +fastjsonschema==2.19.1 +filelock==3.13.1 +flake8==7.1.0 +flake8-docstrings==1.7.0 +flake8-rst==0.8.0 +flake8-rst-docstrings==0.3.0 +fonttools==4.53.0 +fsspec==2024.6.0 +h5py==3.11.0 +idna==3.7 +imageio==2.34.1 +imagesize==1.4.1 +importlib_metadata==7.1.0 +iniconfig==2.0.0 +installer==0.7.0 +ipython==8.25.0 +isodate==0.6.1 +isort==5.13.2 +jaraco.classes==3.3.1 +jedi==0.19.1 +jeepney==0.8.0 +Jinja2==3.1.4 +jsonschema==4.22.0 +jsonschema-specifications==2023.12.1 +keyring==24.3.0 +kiwisolver==1.4.5 +lazy_loader==0.4 +locket==1.0.0 +m2r2==0.3.3.post2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +matplotlib==3.9.0 +matplotlib-inline==0.1.7 +mccabe==0.7.0 +mdurl==0.1.2 +mistune==0.8.4 +more-itertools==10.2.0 +msgpack==1.0.7 +mypy-extensions==1.0.0 +netCDF4==1.7.1 +networkx==3.3 +numcodecs==0.12.1 +numexpr==2.8.4 +numpy==1.26.4 +packaging==24.1 +pandas==2.2.2 +parso==0.8.4 +partd==1.4.2 +pathspec==0.12.1 +pbr==6.0.0 +pexpect==4.9.0 +pillow==10.3.0 +pipenv==2023.12.1 +pixelmatch==0.3.0 +pkginfo==1.9.6 +platformdirs==3.11.0 +pluggy==1.5.0 +poetry==1.7.1 +poetry-core==1.8.1 +poetry-plugin-export==1.6.0 +pooch==1.8.2 +prettier==0.0.7 +prompt_toolkit==3.0.47 +psutil==5.9.8 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyaml-env==1.2.1 +pycodestyle==2.12.0 +pydata-sphinx-theme==0.15.3 +pydocstyle==6.3.0 +pyerfa==2.0.1.4 +pyflakes==3.2.0 +Pygments==2.18.0 +pyhdf==0.11.4 +pykdtree==1.3.12 +pylint==3.2.3 +pyorbital==1.8.2 +pyparsing==3.1.2 +pyproj==3.6.1 +pyPublicDecompWT==2.8.1.5 +pyresample==1.28.3 +pyshp==2.3.1 +pytest==8.2.2 +pytest-cov==5.0.0 +python-dateutil==2.9.0.post0 +pytz==2024.1 +PyYAML==6.0.1 +rapidfuzz==3.6.1 +rasterio==1.3.10 +referencing==0.35.1 +requests==2.32.3 +requests-toolbelt==1.0.0 +restructuredtext-lint==1.4.0 +rich==13.7.1 +rpds-py==0.18.1 +satpy==0.49.0 +scikit-image==0.23.2 +scipy==1.13.1 +SecretStorage==3.3.3 +shapely==2.0.4 +shellingham==1.5.4 +six==1.16.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +soupsieve==2.5 +Sphinx==7.1.2 +sphinx_design==0.6.0 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-autoprogram==0.1.9 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +stack-data==0.6.3 +stevedore==5.2.0 +tabulate==0.9.0 +tifffile==2024.5.22 +tomli==2.0.1 +tomlkit==0.12.5 +toolz==0.12.1 +traitlets==5.14.3 +trollimage==1.23.2 +trollsift==0.5.1 +trove-classifiers==2024.1.31 +typing_extensions==4.12.2 +tzdata==2024.1 +urllib3==2.2.2 +virtualenv==20.26.2 +wcwidth==0.2.13 +xarray==2024.2.0 +xarray-datatree==0.0.14 +zarr==2.18.2 +zipp==3.19.2 diff --git a/environments/pip_base_requirements_1.13.1_20240704.txt b/environments/pip_base_requirements_1.13.1_20240704.txt new file mode 100644 index 000000000..6ff869b99 --- /dev/null +++ b/environments/pip_base_requirements_1.13.1_20240704.txt @@ -0,0 +1,155 @@ +accessible-pygments==0.0.5 +affine==2.4.0 +alabaster==0.7.16 +alphashape==1.3.1 +asciitree==0.3.3 +astroid==3.2.2 +astropy==6.1.1 +astropy-iers-data==0.2024.7.1.0.34.3 +asttokens==2.4.1 +attrs==23.2.0 +Babel==2.15.0 +bandit==1.7.9 +beautifulsoup4==4.12.3 +black==24.4.2 +Cartopy==0.23.0 +certifi==2024.7.4 +cftime==1.6.4 +charset-normalizer==3.3.2 +click==8.1.7 +click-log==0.4.0 +click-plugins==1.1.1 +cligj==0.7.2 +cloudpickle==3.0.0 +colorama==0.4.6 +configobj==5.0.8 +contourpy==1.2.1 +coverage==7.5.4 +cycler==0.12.1 +dask==2024.6.2 +decorator==5.1.1 +dill==0.3.8 +doc8==1.1.1 +docutils==0.20.1 +donfig==0.8.1.post1 +ephem==4.1.5 +exceptiongroup==1.2.1 +executing==2.0.1 +fasteners==0.19 +flake8==7.1.0 +flake8-docstrings==1.7.0 +flake8-rst==0.8.0 +flake8-rst-docstrings==0.3.0 +fonttools==4.53.0 +fsspec==2024.6.1 +h5py==3.11.0 +idna==3.7 +imageio==2.34.2 +imagesize==1.4.1 +importlib_metadata==8.0.0 +iniconfig==2.0.0 +ipython==8.26.0 +isodate==0.6.1 +isort==5.13.2 +jedi==0.19.1 +Jinja2==3.1.4 +jsonschema==4.22.0 +jsonschema-specifications==2023.12.1 +kiwisolver==1.4.5 +lazy_loader==0.4 +locket==1.0.0 +m2r2==0.3.3.post2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +matplotlib==3.9.1 +matplotlib-inline==0.1.7 +mccabe==0.7.0 +mdurl==0.1.2 +mistune==0.8.4 +mypy-extensions==1.0.0 +netCDF4==1.6.5 +networkx==3.3 +numcodecs==0.12.1 +numexpr==2.8.4 +numpy==1.26.4 +packaging==24.1 +pandas==2.2.2 +parso==0.8.4 +partd==1.4.2 +pathspec==0.12.1 +pbr==6.0.0 +pexpect==4.9.0 +pillow==10.4.0 +pixelmatch==0.3.0 +platformdirs==4.2.2 +pluggy==1.5.0 +pooch==1.8.2 +prettier==0.0.7 +prompt_toolkit==3.0.47 +psutil==6.0.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyaml-env==1.2.1 +pycodestyle==2.12.0 +pydata-sphinx-theme==0.15.4 +pydocstyle==6.3.0 +pyerfa==2.0.1.4 +pyflakes==3.2.0 +Pygments==2.18.0 +pyhdf==0.11.4 +pykdtree==1.3.12 +pylint==3.2.5 +pyorbital==1.8.3 +pyparsing==3.1.2 +pyproj==3.6.1 +pyPublicDecompWT==2.8.1.5 +pyresample==1.28.4 +pyshp==2.3.1 +pytest==8.2.2 +pytest-cov==5.0.0 +python-dateutil==2.9.0.post0 +pytz==2024.1 +PyYAML==6.0.1 +rasterio==1.3.10 +referencing==0.35.1 +requests==2.32.3 +restructuredtext_lint==1.4.0 +rich==13.7.1 +rpds-py==0.18.1 +Rtree==1.2.0 +satpy==0.49.0 +scikit-image==0.24.0 +scipy==1.14.0 +shapely==2.0.4 +six==1.16.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +soupsieve==2.5 +Sphinx==7.1.2 +sphinx_design==0.6.0 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-autoprogram==0.1.9 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +stack-data==0.6.3 +stevedore==5.2.0 +tabulate==0.9.0 +tifffile==2024.7.2 +tomli==2.0.1 +tomlkit==0.12.5 +toolz==0.12.1 +traitlets==5.14.3 +trimesh==4.4.1 +trollimage==1.24.0 +trollsift==0.5.1 +typing_extensions==4.12.2 +tzdata==2024.1 +urllib3==2.2.2 +wcwidth==0.2.13 +xarray==2024.2.0 +xarray-datatree==0.0.14 +zarr==2.18.2 +zipp==3.19.2 diff --git a/environments/pip_base_requirements_1.14.0_20240915.txt b/environments/pip_base_requirements_1.14.0_20240915.txt new file mode 100644 index 000000000..9d1f44a1e --- /dev/null +++ b/environments/pip_base_requirements_1.14.0_20240915.txt @@ -0,0 +1,196 @@ +# Package: geoips_base +# Total run time: 156 seconds +# Number data types run: 3 +# Number data types failed: 0 +# Tue Sep 17 01:45:03 UTC 2024 +accessible-pygments==0.0.5 +affine==2.4.0 +alabaster==0.7.16 +alphashape==1.3.1 +annotated-types==0.7.0 +asciitree==0.3.3 +astroid==3.2.4 +astropy==6.1.3 +astropy-iers-data==0.2024.9.12.13.29.57 +asttokens==2.4.1 +attrs==24.2.0 +babel==2.16.0 +bandit==1.7.9 +beautifulsoup4==4.12.3 +black==24.8.0 +brassy==0.0.2.post2 +CacheControl==0.13.1 +Cartopy==0.23.0 +certifi==2024.8.30 +cffi==1.17.1 +cftime==1.6.4 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +click-log==0.4.0 +click-plugins==1.1.1 +cligj==0.7.2 +cloudpickle==3.0.0 +colorama==0.4.6 +configobj==5.0.8 +contourpy==1.3.0 +coverage==7.6.1 +crashtest==0.4.1 +cycler==0.12.1 +dask==2024.9.0 +decorator==5.1.1 +dill==0.3.8 +distlib==0.3.8 +doc8==1.1.2 +docutils==0.20.1 +donfig==0.8.1.post1 +dulwich==0.21.7 +ephem==4.1.5 +exceptiongroup==1.2.2 +executing==2.1.0 +fasteners==0.19 +fastjsonschema==2.19.1 +filelock==3.13.1 +flake8==7.1.1 +flake8-docstrings==1.7.0 +flake8-rst==0.8.0 +flake8-rst-docstrings==0.3.0 +fonttools==4.53.1 +fsspec==2024.9.0 +h5py==3.11.0 +idna==3.9 +imageio==2.35.1 +imagesize==1.4.1 +importlib_metadata==8.5.0 +iniconfig==2.0.0 +installer==0.7.0 +ipython==8.27.0 +isodate==0.6.1 +isort==5.13.2 +jaraco.classes==3.3.1 +jedi==0.19.1 +jeepney==0.8.0 +Jinja2==3.1.4 +jsonschema==4.23.0 +jsonschema-specifications==2023.12.1 +keyring==24.3.0 +kiwisolver==1.4.7 +lazy_loader==0.4 +locket==1.0.0 +m2r2==0.3.3.post2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +matplotlib==3.9.2 +matplotlib-inline==0.1.7 +mccabe==0.7.0 +mdurl==0.1.2 +mistune==0.8.4 +more-itertools==10.2.0 +morecantile==5.4.2 +msgpack==1.0.7 +mypy-extensions==1.0.0 +netCDF4==1.7.1.post2 +networkx==3.3 +numcodecs==0.13.0 +numexpr==2.8.4 +numpy==1.26.4 +packaging==24.1 +pandas==2.2.2 +parso==0.8.4 +partd==1.4.2 +pathspec==0.12.1 +pbr==6.1.0 +pexpect==4.9.0 +pillow==10.4.0 +pinkrst==0.0.1 +pipenv==2023.12.1 +pixelmatch==0.3.0 +pkginfo==1.9.6 +platformdirs==4.3.3 +pluggy==1.5.0 +poetry==1.7.1 +poetry-core==1.8.1 +poetry-plugin-export==1.6.0 +pooch==1.8.2 +prettier==0.0.7 +prompt_toolkit==3.0.47 +psutil==6.0.0 +ptyprocess==0.7.0 +pure_eval==0.2.3 +pyaml-env==1.2.1 +pycodestyle==2.12.1 +pycparser==2.22 +pydantic==2.9.1 +pydantic_core==2.23.3 +pydata-sphinx-theme==0.15.4 +pydocstyle==6.3.0 +pyerfa==2.0.1.4 +pyflakes==3.2.0 +pygit2==1.15.1 +Pygments==2.18.0 +pyhdf==0.11.4 +pykdtree==1.3.13 +pylint==3.2.7 +pyorbital==1.8.3 +pyparsing==3.1.4 +pyproj==3.6.1 +pyPublicDecompWT==2.8.1.6 +pyresample==1.30.0 +pyshp==2.3.1 +pytest==8.3.3 +pytest-cov==5.0.0 +python-dateutil==2.9.0.post0 +pytz==2024.2 +PyYAML==6.0.2 +rapidfuzz==3.6.1 +rasterio==1.3.11 +referencing==0.35.1 +requests==2.32.3 +requests-toolbelt==1.0.0 +restructuredtext-lint==1.4.0 +rich==13.8.1 +rich-argparse==1.5.2 +rio-cogeo==5.3.4 +rpds-py==0.20.0 +Rtree==1.3.0 +satpy==0.51.0 +scikit-image==0.24.0 +scipy==1.14.1 +SecretStorage==3.3.3 +shapely==2.0.6 +shellingham==1.5.4 +six==1.16.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +soupsieve==2.6 +Sphinx==7.1.2 +sphinx-argparse==0.5.2 +sphinx_design==0.6.1 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-autoprogram==0.1.9 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 +stack-data==0.6.3 +stevedore==5.3.0 +tabulate==0.9.0 +tifffile==2024.8.30 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +traitlets==5.14.3 +trimesh==4.4.9 +trollimage==1.25.0 +trollsift==0.5.1 +trove-classifiers==2024.1.31 +typing_extensions==4.12.2 +tzdata==2024.1 +urllib3==2.2.3 +virtualenv==20.26.3 +wcwidth==0.2.13 +xarray==2024.2.0 +xarray-datatree==0.0.14 +zarr==2.18.3 +zipp==3.20.2 diff --git a/environments/pip_full_requirements_1.14.0_20240916.txt b/environments/pip_full_requirements_1.14.0_20240916.txt new file mode 100644 index 000000000..097204d35 --- /dev/null +++ b/environments/pip_full_requirements_1.14.0_20240916.txt @@ -0,0 +1,197 @@ +# Package: geoips_full +# Total run time: 2558 seconds +# Number data types run: 56 +# Number data types failed: 0 +# Mon Sep 16 22:06:05 UTC 2024 +accessible-pygments==0.0.5 +affine==2.4.0 +# Editable install with no version control (akima86==0.1.1) +alabaster==0.7.16 +alphashape==1.3.1 +annotated-types==0.7.0 +asciitree==0.3.3 +astroid==3.2.4 +astropy==6.1.3 +astropy-iers-data==0.2024.9.12.13.29.57 +asttokens==2.4.1 +attrs==24.2.0 +babel==2.16.0 +bandit==1.7.9 +beautifulsoup4==4.12.3 +black==24.8.0 +brassy==0.0.2.post2 +CacheControl==0.13.1 +Cartopy==0.23.0 +certifi==2024.8.30 +cffi==1.17.1 +cftime==1.6.4 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +click-log==0.4.0 +click-plugins==1.1.1 +cligj==0.7.2 +cloudpickle==3.0.0 +colorama==0.4.6 +configobj==5.0.8 +contourpy==1.3.0 +coverage==7.6.1 +crashtest==0.4.1 +cycler==0.12.1 +dask==2024.9.0 +decorator==5.1.1 +dill==0.3.8 +distlib==0.3.8 +doc8==1.1.2 +docutils==0.20.1 +donfig==0.8.1.post1 +dulwich==0.21.7 +ephem==4.1.5 +exceptiongroup==1.2.2 +executing==2.1.0 +fasteners==0.19 +fastjsonschema==2.19.1 +filelock==3.13.1 +flake8==7.1.1 +flake8-docstrings==1.7.0 +flake8-rst==0.8.0 +flake8-rst-docstrings==0.3.0 +fonttools==4.53.1 +fsspec==2024.9.0 +h5py==3.11.0 +idna==3.9 +imageio==2.35.1 +imagesize==1.4.1 +importlib_metadata==8.5.0 +iniconfig==2.0.0 +installer==0.7.0 +ipython==8.27.0 +isodate==0.6.1 +isort==5.13.2 +jaraco.classes==3.3.1 +jedi==0.19.1 +jeepney==0.8.0 +Jinja2==3.1.4 +jsonschema==4.23.0 +jsonschema-specifications==2023.12.1 +keyring==24.3.0 +kiwisolver==1.4.7 +lazy_loader==0.4 +locket==1.0.0 +m2r2==0.3.3.post2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +matplotlib==3.9.2 +matplotlib-inline==0.1.7 +mccabe==0.7.0 +mdurl==0.1.2 +mistune==0.8.4 +more-itertools==10.2.0 +morecantile==5.4.2 +msgpack==1.0.7 +mypy-extensions==1.0.0 +netCDF4==1.7.1.post2 +networkx==3.3 +numcodecs==0.13.0 +numexpr==2.8.4 +numpy==1.26.4 +packaging==24.1 +pandas==2.2.2 +parso==0.8.4 +partd==1.4.2 +pathspec==0.12.1 +pbr==6.1.0 +pexpect==4.9.0 +pillow==10.4.0 +pinkrst==0.0.1 +pipenv==2023.12.1 +pixelmatch==0.3.0 +pkginfo==1.9.6 +platformdirs==4.3.3 +pluggy==1.5.0 +poetry==1.7.1 +poetry-core==1.8.1 +poetry-plugin-export==1.6.0 +pooch==1.8.2 +prettier==0.0.7 +prompt_toolkit==3.0.47 +psutil==6.0.0 +ptyprocess==0.7.0 +pure_eval==0.2.3 +pyaml-env==1.2.1 +pycodestyle==2.12.1 +pycparser==2.22 +pydantic==2.9.1 +pydantic_core==2.23.3 +pydata-sphinx-theme==0.15.4 +pydocstyle==6.3.0 +pyerfa==2.0.1.4 +pyflakes==3.2.0 +pygit2==1.15.1 +Pygments==2.18.0 +pyhdf==0.11.4 +pykdtree==1.3.13 +pylint==3.2.7 +pyorbital==1.8.3 +pyparsing==3.1.4 +pyproj==3.6.1 +pyPublicDecompWT==2.8.1.6 +pyresample==1.30.0 +pyshp==2.3.1 +pytest==8.3.3 +pytest-cov==5.0.0 +python-dateutil==2.9.0.post0 +pytz==2024.2 +PyYAML==6.0.2 +rapidfuzz==3.6.1 +rasterio==1.3.11 +referencing==0.35.1 +requests==2.32.3 +requests-toolbelt==1.0.0 +restructuredtext-lint==1.4.0 +rich==13.8.1 +rich-argparse==1.5.2 +rio-cogeo==5.3.4 +rpds-py==0.20.0 +Rtree==1.3.0 +satpy==0.51.0 +scikit-image==0.24.0 +scipy==1.14.1 +SecretStorage==3.3.3 +shapely==2.0.6 +shellingham==1.5.4 +six==1.16.0 +snowballstemmer==2.2.0 +snuggs==1.4.7 +soupsieve==2.6 +Sphinx==7.1.2 +sphinx-argparse==0.5.2 +sphinx_design==0.6.1 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-autoprogram==0.1.9 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 +stack-data==0.6.3 +stevedore==5.3.0 +tabulate==0.9.0 +tifffile==2024.8.30 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +traitlets==5.14.3 +trimesh==4.4.9 +trollimage==1.25.0 +trollsift==0.5.1 +trove-classifiers==2024.1.31 +typing_extensions==4.12.2 +tzdata==2024.1 +urllib3==2.2.3 +virtualenv==20.26.3 +wcwidth==0.2.13 +xarray==2024.2.0 +xarray-datatree==0.0.14 +zarr==2.18.3 +zipp==3.20.2 diff --git a/geoips/commandline/geoips_tree.py b/geoips/commandline/geoips_tree.py index 946c980bb..5881e094b 100644 --- a/geoips/commandline/geoips_tree.py +++ b/geoips/commandline/geoips_tree.py @@ -1,3 +1,6 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + """GeoIPS CLI "tree" command. Single 'geoips tree' command which will display GeoIPS CLI commands up to a --max-depth diff --git a/geoips/commandline/log_setup.py b/geoips/commandline/log_setup.py index 8f85cccb1..d34f6bfd5 100644 --- a/geoips/commandline/log_setup.py +++ b/geoips/commandline/log_setup.py @@ -8,10 +8,6 @@ from textwrap import wrap -logging_setup_done = False -log = None - - def log_with_emphasis(print_func, *messages): """Print messages boxed in asterisks using the specified print function. @@ -190,12 +186,6 @@ def setup_logging(logging_level=None, verbose=True): instance via `LOG = logging.getLogger(__name__)` and it will behave the same as the root logger. - If this function was called previously in the program's traceback in any other - location, then setup_logging will default to the log setup in the very first call. - This is done so log statements aren't duplicated the number of times this function - has been called. Therefore, it is smart to ensure that the first call to this - function is the lowest log level you'ld like to capture. - Parameters ---------- logging_level : str, default=None @@ -206,34 +196,21 @@ def setup_logging(logging_level=None, verbose=True): Determines which log formatter will be used. If `True`, a longer format will be used, providing more information, but also cluttering the screen. If `False`, a shorter format will be used. - - Globals - ------- - logging_setup_done: bool - - Whether or not this function has already been called - log: logging.RootLogger - - A root logging object that has the GeoIPS log format applied. This will only - be set once in a program's execution. """ - global logging_setup_done, log - - if not logging_setup_done: - log = logging.getLogger() - # If logging_level was not specified, default to INTERACTIVE here. - if not logging_level: - logging_level = "INTERACTIVE" - log.setLevel(getattr(logging, logging_level)) - fmt = logging.Formatter( - "%(asctime)s %(module)12s.py:%(lineno)-4d %(levelname)7s: %(message)s", - "%d_%H%M%S", - ) - if not verbose: - fmt = logging.Formatter("%(asctime)s: %(message)s", "%d_%H%M%S") + log = logging.getLogger() + # If logging_level was not specified, default to INTERACTIVE here. + if not logging_level: + logging_level = "INTERACTIVE" + log.setLevel(getattr(logging, logging_level)) + fmt = logging.Formatter( + "%(asctime)s %(module)12s.py:%(lineno)-4d %(levelname)7s: %(message)s", + "%d_%H%M%S", + ) + if not verbose: + fmt = logging.Formatter("%(asctime)s: %(message)s", "%d_%H%M%S") + if not log.handlers: stream_hndlr = logging.StreamHandler(sys.stdout) stream_hndlr.setFormatter(fmt) stream_hndlr.setLevel(logging.INFO) log.addHandler(stream_hndlr) - - logging_setup_done = True - return log diff --git a/geoips/dev/product.py b/geoips/dev/product.py index 396809197..402a6ec6d 100644 --- a/geoips/dev/product.py +++ b/geoips/dev/product.py @@ -140,6 +140,32 @@ def get_data_range(prod_plugin, output_dict=None): return alg_args["output_data_range"].copy() +def get_cmap_data_range(prod_plugin, output_dict=None): + """Interface will be deprecated v2.0. + + Retrieve required data range for requested product + + Parameters + ---------- + product_name : str + Name of requested product (ie, 'IR-BD', '89H', 'color89Nearest', etc) + + Returns + ------- + data_range : list + List of float specifying min and max value for the output product + ``.colormapper..arguments['data_range']`` + """ + cmap_args = prod_plugin["spec"]["colormapper"]["plugin"]["arguments"] + if "data_range" not in cmap_args: + raise TypeError( + f"Product {prod_plugin.name} does not define 'output_data_range' for its " + f"algorithm." + ) + # Add .copy() so we aren't returning the ACTUAL list attached to the args + return cmap_args["data_range"].copy() + + def get_product_display_name(prod_plugin, output_dict=None): """Interface will be deprecated v2.0. diff --git a/geoips/filenames/base_paths.py b/geoips/filenames/base_paths.py index 306ef2c65..70e0cc13d 100755 --- a/geoips/filenames/base_paths.py +++ b/geoips/filenames/base_paths.py @@ -1,252 +1,185 @@ # # # This source code is protected under the license referenced at # # # https://github.com/NRLMMD-GEOIPS. -"""Collection of base path names used throughout GeoIPS. +"""Module for setting paths used throughout GeoIPS. -Everything defaults to subdirectories relative to the -REQUIRED environment variable GEOIPS_OUTDIRS. +This module sets various directory paths required for GeoIPS. +Setting these variables here keeps geoips code and outputs consolidated. +Eventually, we aim to supplant this method of setting variables with a config file. +`GEOIPS_OUTDIRS` serves as the base reference for all other variables. +It defaults to the values set in environment variables. +It also provides a function for creating directories. -Individual GEOIPS_OUTDIRS relative paths can be overridden -by setting appropriate environment variables. -""" - -# Python Standard Libraries -import logging -from os import getenv -from os.path import exists, dirname, join as pathjoin -import socket - -LOG = logging.getLogger(__name__) - -PATHS = {} -# Get the base package directory -PATHS["BASE_PATH"] = pathjoin(dirname(__file__), "..") -# URL of the most GeoIPS GitHub hosted documentation. -PATHS["GEOIPS_DOCS_URL"] = r"https://nrlmmd-geoips.github.io/geoips/" +Functions +--------- +- get_env_var: Retrieve an environment variable or a provided default value. +- initialize_paths: Returns a dictionary with GeoIPS directories and variables. +- make_dirs: Create directories if they don't already exist. -PATHS["GEOIPS_OPERATIONAL_USER"] = False -if getenv("GEOIPS_OPERATIONAL_USER"): - PATHS["GEOIPS_OPERATIONAL_USER"] = getenv("GEOIPS_OPERATIONAL_USER") +Attributes +---------- +- PATHS: Dictionary with initialized paths for various GeoIPS directories and URLs. -# At a minimum, GEOIPS_OUTDIRS must be defined. -if not getenv("GEOIPS_OUTDIRS"): - raise KeyError( - "GEOIPS_OUTDIRS must be set in your environment. " - "Please set GEOIPS_OUTDIRS and try again" - ) -else: - PATHS["GEOIPS_OUTDIRS"] = getenv("GEOIPS_OUTDIRS").rstrip("/") - -if getenv("GEOIPS_PACKAGES_DIR") and exists(getenv("GEOIPS_PACKAGES_DIR")): - PATHS["GEOIPS_PACKAGES_DIR"] = getenv("GEOIPS_PACKAGES_DIR").rstrip("/") -elif getenv("GEOIPS_PACKAGES_DIR"): - PATHS["GEOIPS_PACKAGES_DIR"] = getenv("GEOIPS_PACKAGES_DIR").rstrip("/") -else: - PATHS["GEOIPS_PACKAGES_DIR"] = pathjoin(PATHS["BASE_PATH"], "..", "..") - -if not getenv("GEOIPS_BASEDIR"): - PATHS["GEOIPS_BASEDIR"] = pathjoin(PATHS["GEOIPS_PACKAGES_DIR"], "..") -else: - PATHS["GEOIPS_BASEDIR"] = getenv("GEOIPS_BASEDIR").rstrip("/") +Environment Variables: +- GEOIPS_OUTDIRS (required): Base output directory for GeoIPS. +""" -if getenv("GEOIPS_TESTDATA_DIR"): - PATHS["GEOIPS_TESTDATA_DIR"] = getenv("GEOIPS_TESTDATA_DIR") -else: - PATHS["GEOIPS_TESTDATA_DIR"] = PATHS["GEOIPS_BASEDIR"] + "/test_data" +import logging +import os +import socket -if getenv("GEOIPS_DEPENDENCIES_DIR"): - PATHS["GEOIPS_DEPENDENCIES_DIR"] = getenv("GEOIPS_DEPENDENCIES_DIR") -else: - PATHS["GEOIPS_DEPENDENCIES_DIR"] = PATHS["GEOIPS_BASEDIR"] + "/geoips_dependencies" -# Location for writing out presectored data files, but unregistered -if getenv("PRESECTORED_DATA_PATH"): - PATHS["PRESECTORED_DATA_PATH"] = getenv("PRESECTORED_DATA_PATH").rstrip("/") -else: - PATHS["PRESECTORED_DATA_PATH"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "preprocessed", "sectored" - ) +def get_env_var(var_name, default, rstrip_path=True): + """Retrieve environment variable or provided default, optionally rstrip a '/'. -# Location for writing out preread, but unsectored/registered, data files -if getenv("PREREAD_DATA_PATH"): - PATHS["PREREAD_DATA_PATH"] = getenv("PREREAD_DATA_PATH").rstrip("/") -else: - PATHS["PREREAD_DATA_PATH"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "preprocessed", "unsectored" - ) + Parameters + ---------- + var_name : str + The name of the environment variable. + default : str + The default value to return if the environment variable is not set. + rstrip_path : bool, optional + If True, strip trailing slashes from the returned path (default is True). -# Location for writing out preregistered data files, but no algorithms applied -if getenv("PREREGISTERED_DATA_PATH"): - PATHS["PREREGISTERED_DATA_PATH"] = getenv("PREREGISTERED_DATA_PATH").rstrip("/") -else: - PATHS["PREREGISTERED_DATA_PATH"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "preprocessed", "registered" - ) + Returns + ------- + str + The value of the environment variable or the default value if not set. + """ + env_value = os.getenv(var_name) + if env_value: + if rstrip_path: + return env_value.rstrip("/") + else: + return env_value + else: + return default -# Location for writing out precalculated data files (algorithms applied) -if getenv("PRECALCULATED_DATA_PATH"): - PATHS["PRECALCULATED_DATA_PATH"] = getenv("PRECALCULATED_DATA_PATH").rstrip("/") -else: - PATHS["PRECALCULATED_DATA_PATH"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "preprocessed", "algorithms" - ) -# Location for writing out pregenerated "clean" imagery files -if getenv("CLEAN_IMAGERY_PATH"): - PATHS["CLEAN_IMAGERY_PATH"] = getenv("CLEAN_IMAGERY_PATH").rstrip("/") -else: - PATHS["CLEAN_IMAGERY_PATH"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "preprocessed", "clean_imagery" - ) +def initialize_paths(): + """ + Initialize and return a dictionary of paths used throughout GeoIPS. -# Location for writing out pregenerated "clean" imagery files -if getenv("ANNOTATED_IMAGERY_PATH"): - PATHS["ANNOTATED_IMAGERY_PATH"] = getenv("ANNOTATED_IMAGERY_PATH").rstrip("/") -else: - PATHS["ANNOTATED_IMAGERY_PATH"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "preprocessed", "annotated_imagery" - ) + Returns + ------- + dict + A dictionary containing various paths used by GeoIPS. -# Location for writing out precalculated data files (algorithms applied) -if getenv("FINAL_DATA_PATH"): - PATHS["FINAL_DATA_PATH"] = getenv("FINAL_DATA_PATH").rstrip("/") -else: - PATHS["FINAL_DATA_PATH"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "preprocessed", "final" - ) + Raises + ------ + KeyError + If the required environment variable GEOIPS_OUTDIRS is not set. + """ + paths = {} -# Location for writing out pregenerated geolocation netcdf files -if getenv("PREGENERATED_GEOLOCATION_PATH"): - PATHS["PREGENERATED_GEOLOCATION_PATH"] = getenv( - "PREGENERATED_GEOLOCATION_PATH" - ).rstrip("/") -else: - PATHS["PREGENERATED_GEOLOCATION_PATH"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "preprocessed", "geolocation" + # Base and Documentation Paths + paths["BASE_PATH"] = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir) ) -# GEOIPS_COPYRIGHT determines what organization name displays in imagery titles, etc. -PATHS["GEOIPS_COPYRIGHT"] = "NRL-Monterey" -if getenv("GEOIPS_COPYRIGHT"): - PATHS["GEOIPS_COPYRIGHT"] = getenv("GEOIPS_COPYRIGHT") - -# GEOIPS_COPYRIGHT_ABBREVIATED provides an abbreviated version of the -# copyright, best for filenames, etc. -PATHS["GEOIPS_COPYRIGHT_ABBREVIATED"] = "NRLMRY" -if getenv("GEOIPS_COPYRIGHT_ABBREVIATED"): - PATHS["GEOIPS_COPYRIGHT_ABBREVIATED"] = getenv("GEOIPS_COPYRIGHT_ABBREVIATED") - -PATHS["GEOIPS_RCFILE"] = "" -if getenv("GEOIPS_RCFILE"): - PATHS["GEOIPS_RCFILE"] = getenv("GEOIPS_RCFILE") - -PATHS["TC_TEMPLATE"] = pathjoin( - PATHS["BASE_PATH"], "plugins", "yaml", "sectors", "dynamic", "tc_web_template.yaml" -) -if getenv("TC_TEMPLATE"): - PATHS["TC_TEMPLATE"] = getenv("TC_TEMPLATE") - -PATHS["DEFAULT_QUEUE"] = None -if getenv("DEFAULT_QUEUE"): - PATHS["DEFAULT_QUEUE"] = getenv("DEFAULT_QUEUE") - -PATHS["BOXNAME"] = socket.gethostname() -if not getenv("HOME"): - # Windows - PATHS["HOME"] = getenv("HOMEDRIVE") + getenv("HOMEPATH") -else: - PATHS["HOME"] = getenv("HOME").rstrip("/") - -if not getenv("SCRATCH"): - PATHS["SCRATCH"] = pathjoin(getenv("GEOIPS_OUTDIRS"), "scratch") -else: - PATHS["SCRATCH"] = getenv("SCRATCH").rstrip("/") - -if not getenv("LOCALSCRATCH"): - PATHS["LOCALSCRATCH"] = PATHS["SCRATCH"] -else: - PATHS["LOCALSCRATCH"] = getenv("LOCALSCRATCH").rstrip("/") - -if not getenv("SHAREDSCRATCH"): - PATHS["SHAREDSCRATCH"] = PATHS["SCRATCH"] -else: - PATHS["SHAREDSCRATCH"] = getenv("SHAREDSCRATCH").rstrip("/") - -if not getenv("GEOIPS_ANCILDAT"): - PATHS["GEOIPS_ANCILDAT"] = pathjoin(PATHS["GEOIPS_OUTDIRS"], "ancildat") -elif getenv("GEOIPS_ANCILDAT"): - PATHS["GEOIPS_ANCILDAT"] = getenv("GEOIPS_ANCILDAT").rstrip("/") -# Separating the auto-generated files from the source files allows for -# individual users to read from the shared ancildat, and write to their own -# auto-generated location -if not getenv("GEOIPS_ANCILDAT_AUTOGEN"): - PATHS["GEOIPS_ANCILDAT_AUTOGEN"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "ancildat_autogen" + # Output Directories + if not os.getenv("GEOIPS_OUTDIRS"): + raise KeyError( + "GEOIPS_OUTDIRS must be set in your environment. " + "Please set GEOIPS_OUTDIRS and try again" + ) + paths["GEOIPS_OUTDIRS"] = os.getenv("GEOIPS_OUTDIRS").rstrip("/") + paths["GEOIPS_PACKAGES_DIR"] = os.path.abspath( + os.path.join(paths["BASE_PATH"], os.pardir, os.pardir) ) -elif getenv("GEOIPS_ANCILDAT_AUTOGEN"): - PATHS["GEOIPS_ANCILDAT_AUTOGEN"] = getenv("GEOIPS_ANCILDAT_AUTOGEN").rstrip("/") - -if not getenv("LOGDIR"): - PATHS["LOGDIR"] = pathjoin(PATHS["GEOIPS_OUTDIRS"], "logs") -else: - PATHS["LOGDIR"] = getenv("LOGDIR").rstrip("/") - -if not getenv("GEOIPSDATA"): - PATHS["GEOIPSDATA"] = pathjoin(PATHS["GEOIPS_OUTDIRS"], "geoipsdata") -else: - PATHS["GEOIPSDATA"] = getenv("GEOIPSDATA").rstrip("/") - -if not getenv("GEOIPS_VERS"): - PATHS["GEOIPS_VERS"] = "0.0.0" -else: - PATHS["GEOIPS_VERS"] = getenv("GEOIPS_VERS") - - -if getenv("TCWWW"): - PATHS["TCWWW"] = getenv("TCWWW").rstrip("/") -else: - PATHS["TCWWW"] = pathjoin(PATHS["GEOIPS_OUTDIRS"], "preprocessed", "tcwww") - -if getenv("TCWWW_URL"): - PATHS["TCWWW_URL"] = getenv("TCWWW_URL").rstrip("/") -else: - PATHS["TCWWW_URL"] = PATHS["TCWWW"] - -if getenv("PUBLICWWW"): - PATHS["PUBLICWWW"] = getenv("PUBLICWWW").rstrip("/") -else: - PATHS["PUBLICWWW"] = pathjoin(PATHS["GEOIPS_OUTDIRS"], "preprocessed", "publicwww") - -if getenv("PUBLICWWW_URL"): - PATHS["PUBLICWWW_URL"] = getenv("PUBLICWWW_URL").rstrip("/") -else: - PATHS["PUBLICWWW_URL"] = PATHS["PUBLICWWW"] - -if getenv("PRIVATEWWW"): - PATHS["PRIVATEWWW"] = getenv("PRIVATEWWW").rstrip("/") -else: - PATHS["PRIVATEWWW"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "preprocessed", "privatewww" + paths["GEOIPS_BASEDIR"] = get_env_var( + "GEOIPS_BASEDIR", + os.path.abspath(os.path.join(paths["GEOIPS_PACKAGES_DIR"], os.pardir)), ) -if getenv("PRIVATEWWW_URL"): - PATHS["PRIVATEWWW_URL"] = getenv("PRIVATEWWW_URL").rstrip("/") -else: - PATHS["PRIVATEWWW_URL"] = PATHS["PRIVATEWWW"] - -PATHS["TC_DECKS_DB"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "longterm_files", "tc", "tc_decks.db" -) -if getenv("TC_DECKS_DB"): - PATHS["TC_DECKS_DB"] = getenv("TC_DECKS_DB") - -if getenv("TC_DECKS_DIR"): - PATHS["TC_DECKS_DIR"] = getenv("TC_DECKS_DIR").rstrip("/") -else: - PATHS["TC_DECKS_DIR"] = pathjoin( - PATHS["GEOIPS_OUTDIRS"], "longterm_files", "tc", "decks" - ) + geoips_global_variables = { + # GeoIPS Documentation URL + "GEOIPS_DOCS_URL": r"https://nrlmmd-geoips.github.io/geoips/", + # Version + "GEOIPS_VERS": os.getenv("GEOIPS_VERS", "0.0.0"), + # Operational User + "GEOIPS_OPERATIONAL_USER": os.getenv("GEOIPS_OPERATIONAL_USER", False), + # Copyright Information + "GEOIPS_COPYRIGHT": os.getenv("GEOIPS_COPYRIGHT", "NRL-Monterey"), + "GEOIPS_COPYRIGHT_ABBREVIATED": os.getenv( + "GEOIPS_COPYRIGHT_ABBREVIATED", "NRLMRY" + ), + # Configuration and Queue + "GEOIPS_RCFILE": os.getenv("GEOIPS_RCFILE", ""), + "DEFAULT_QUEUE": os.getenv("DEFAULT_QUEUE", None), + # Computer Identifier + "BOXNAME": socket.gethostname(), + "OUTPUT_CHECKER_THRESHOLD_IMAGE": float( + os.getenv("OUTPUT_CHECKER_THRESHOLD_IMAGE", 0.05) + ), + } + + # these are the defaults for path based environment variables + # that default to locations under paths set above + # They can are overridden by set environment variables. + default_derivative_directory_paths = { + paths["GEOIPS_BASEDIR"]: { + "GEOIPS_TESTDATA_DIR": "test_data", + "GEOIPS_DEPENDENCIES_DIR": "geoips_dependencies", + }, + paths["GEOIPS_OUTDIRS"]: { + # Data Output Paths + "PRESECTORED_DATA_PATH": "preprocessed/sectored", + "PREREAD_DATA_PATH": "preprocessed/unsectored", + "PREREGISTERED_DATA_PATH": "preprocessed/registered", + "PRECALCULATED_DATA_PATH": "preprocessed/algorithms", + "CLEAN_IMAGERY_PATH": "preprocessed/clean_imagery", + "ANNOTATED_IMAGERY_PATH": "preprocessed/annotated_imagery", + "FINAL_DATA_PATH": "preprocessed/final", + "PREGENERATED_GEOLOCATION_PATH": "preprocessed/geolocation", + # Scratch Directories + "SCRATCH": "scratch", + "LOCALSCRATCH": "scratch", + "SHAREDSCRATCH": "scratch", + # Log and Data Directories + "LOGDIR": "logs", + "GEOIPSDATA": "geoipsdata", + # Ancillary Data Directories + "GEOIPS_ANCILDAT_AUTOGEN": "ancildat_autogen", + "GEOIPS_ANCILDAT": "ancildat", + # WWW Paths + "TCWWW": "preprocessed/tcwww", + "PUBLICWWW": "preprocessed/publicwww", + "PRIVATEWWW": "preprocessed/privatewww", + # Tropical Cyclone Paths + "TC_DECKS_DB": "longterm_files/tc/tc_decks.db", + "TC_DECKS_DIR": "longterm_files/tc/decks", + }, + paths["BASE_PATH"]: { + "TC_TEMPLATE": "plugins/yaml/sectors/dynamic/tc_web_template.yaml", + }, + } + + # looping through all the directory-based paths and global variables set above + # using "get_env_var" function to set the variables to the environment variable + # specified option (when defined via the first argument) + # else defaulting to the passed-in default (second argument) + for key, value in geoips_global_variables.items(): + paths[key] = get_env_var(key, value) + + for top_directory, sub_directories in default_derivative_directory_paths.items(): + for key, sub_path in sub_directories.items(): + paths[key] = get_env_var(key, os.path.join(top_directory, sub_path)) + + # Handling special cases now: home for linux/windows + if not os.getenv("HOME"): + # need home drive default for windows + paths["HOME"] = os.getenv("HOMEDRIVE") + os.getenv("HOMEPATH") + else: + paths["HOME"] = os.getenv("HOME").rstrip("/") + + # Setting links for WWW Paths + www_paths = ["TCWWW", "PUBLICWWW", "PRIVATEWWW"] + for path in www_paths: + paths[f"{path}_URL"] = get_env_var(f"{path}_URL", paths[path]) + + return paths def make_dirs(path): @@ -262,17 +195,12 @@ def make_dirs(path): str Path if successfully created """ - from os import makedirs - - if not exists(path): - try: - LOG.info("Creating directory %s", path) - makedirs(path, mode=0o755) - except OSError as resp: - LOG.warning( - "%s: We thought %s did not exist, but then it did. " - "Not trying to make directory", - resp, - path, - ) + LOG.info("Creating directory %s if it doesn't already exist.", path) + os.makedirs(path, mode=0o755, exist_ok=True) return path + + +LOG = logging.getLogger(__name__) + +# Initialize the PATHS dictionary +PATHS = initialize_paths() diff --git a/geoips/geoips_utils.py b/geoips/geoips_utils.py index 77786c1bd..ddcf9fd2b 100644 --- a/geoips/geoips_utils.py +++ b/geoips/geoips_utils.py @@ -10,12 +10,14 @@ from shutil import get_terminal_size import json from tabulate import tabulate +from pathlib import Path # import yaml import logging from importlib import metadata, resources from geoips.errors import PluginRegistryError, PluginPackageNotFoundError +from geoips.filenames.base_paths import PATHS as geoips_paths LOG = logging.getLogger(__name__) @@ -203,21 +205,113 @@ def output_process_times(process_datetimes, num_jobs=None, job_str="GeoIPS 2"): LOG.info(" MISSING Process Time %s: %s", job_str, process_name) +def order_paths_from_least_to_most_specific(paths): + """ + Orders a list of filesystem paths from least to most specific. + + This function takes a list of filesystem paths and returns a new list of paths + ordered from the least specific (higher-level directories) to the most specific + (subdirectories and files). It expands environmental variables in paths. + + Parameters + ---------- + paths : list of str or pathlib.Path + A list of filesystem paths to be ordered. + + Returns + ------- + list of pathlib.Path + A list of filesystem paths ordered from least to most specific. + + Examples + -------- + >>> paths = [ + ... '/home/user/docs/', + ... '/home/user/images/', + ... '/home/user/', + ... '/home/user/images/photo.jpg' + ... '/home/user/docs/report.txt', + ... ] + >>> order_paths_from_least_to_most_specific(paths) + [PosixPath('/home/user/'), + PosixPath('/home/user/docs/'), + PosixPath('/home/user/images/'), + PosixPath('/home/user/docs/report.txt'), + PosixPath('/home/user/images/photo.jpg')] + + """ + if not paths: + return [] + ordered_paths = [] + unordered_paths = [] + paths = [Path(os.path.expandvars(p)) for p in paths] + for i, path in enumerate(paths): + other_paths = paths[:i] + paths[i + 1 :] + if all([path not in other_path.parents for other_path in other_paths]): + # path not in other paths, least specific already + ordered_paths.append(path) + else: + # path in other path, needs more sorting + unordered_paths.append(path) + return ordered_paths + order_paths_from_least_to_most_specific(unordered_paths) + + def replace_geoips_paths_in_list( replace_list, replace_paths=None, base_paths=None, curly_braces=False ): - """Replace geoips paths for every path-based element in a list.""" - newlist = [] + """ + Replace GeoIPS paths with geoips settings in elements of a list. + + This function iterates over each element in the provided `replace_list`, + attempting to replace GeoIPS paths within each element using the + `replace_geoips_paths` function. If an element raises a `TypeError` + when cast to a pathlib path, it is skipped. + + Parameters + ---------- + replace_list : list + A list of elements to process. Elements can be of any type, but only those that + are Path-like will be processed. + replace_paths : dict, optional + Passed to replace_geoips_paths + base_paths : dict, optional + Passed to replace_geoips_paths + curly_braces : bool, optional + Passed to replace_geoips_paths + + Returns + ------- + list + A new list containing the elements with GeoIPS paths replaced where possible. + Elements that could not be processed are included unchanged. + + Examples + -------- + >>> replace_geoips_paths_in_list(['/home/geoips/data/project', + ... 'no_replacement_here']) + ['$GEOIPS_DATA_DIR/project', 'no_replacement_here'] + + See Also + -------- + replace_geoips_paths : Function used to replace GeoIPS paths in individual elements. + """ + new_list = [] # Go through each element in the list for val in replace_list: - # If this element is a str, and contains "/", it's probably a path, - # and we can replace the geoips paths. - if isinstance(val, str) and "/" in val: - newlist += [replace_geoips_paths(val)] - # Otherwise, just put the current element back - else: - newlist += [val] - return newlist + try: + new_list.append( + replace_geoips_paths( + val, + replace_paths=replace_paths, + base_paths=base_paths, + curly_braces=curly_braces, + ) + ) + except TypeError: + # Otherwise, just put the current element back + new_list.append(val) + continue + return new_list def replace_geoips_paths_in_dict( @@ -239,81 +333,102 @@ def replace_geoips_paths_in_dict( def replace_geoips_paths( - fname, + path, replace_paths=None, base_paths=None, curly_braces=False, ): - """Replace standard environment variables with their non-expanded equivalents. + """Replace specified sub-paths in path with related environment variable names. - Ie, replace + This function replaces paths in the provided path with their corresponding + environment variable names. This is useful for generating output paths + or metadata that are independent of specific installation directories. - * ``$HOME/geoproc/geoips_packages with $GEOIPS_PACKAGES_DIR`` - * ``$HOME/geoproc/geoips_outdirs with $GEOIPS_OUTDIRS`` - * ``$HOME/geoproc with $GEOIPS_BASEDIR`` + For example, it can replace: - This allows generating output YAML fields / NetCDF attributes that can match - between different instantiations. + - ``'/home/user/geoproc/geoips_packages'`` with ``'$GEOIPS_PACKAGES_DIR'`` + - ``'/home/user/geoproc/geoips_outdirs'`` with ``'$GEOIPS_OUTDIRS'`` + - ``'/home/user/geoproc'`` with ``'$GEOIPS_BASEDIR'`` Parameters ---------- - fname : str - Full path to a filename on disk - replace_paths : list, default=None - * Explicit list of standard variable names you would like replaced. - * If None, replace - ``['GEOIPS_OUTDIRS', 'GEOIPS_PACKAGES_DIR', 'GEOIPS_TESTDATA_DIR', - 'GEOIPS_DEPENDENCIES_DIR', 'GEOIPS_BASEDIR']`` - base_paths : list, default=None - * List of PATHS dictionaries in which to find the "replace_paths" variables - * If None, use geoips.filenames.base_paths - curly_braces: bool, default=False - * Specifies whether to include curly braces in the environment variables - or not. + path : str or pathlib.Path + The path in which to replace base paths. + replace_paths : list of str, optional + A list of environment variable names whose corresponding paths should be + replaced in `path`. + If `None`, defaults to: + + ``['$GEOIPS_OUTDIRS', '$GEOIPS_PACKAGES_DIR', '$GEOIPS_TESTDATA_DIR', + '$GEOIPS_DEPENDENCIES_DIR', '$GEOIPS_BASEDIR']`` + base_paths : dict, optional + A dictionary mapping environment variable names to their corresponding base + paths. If `None`, defaults to `geoips.filenames.base_paths.PATH`. + curly_braces : bool, default=False + If `True`, includes curly braces in the environment variables + (e.g., ``'${GEOIPS_BASEDIR}'``), + otherwise excludes them (e.g., ``'$GEOIPS_BASEDIR'``). Returns ------- - fname : str - Path to file on disk, with explicit path replaced with environment - variable name and/or full URL. + str + The path with specified base paths replaced with environment variable names. Notes ----- - Note it replaces ALL standard variables that have a corresponding - ``_URL`` variable. - - Additionally, it replaces variables specified in "replace_paths" list with - the unexpanded environment variable name. + The function iterates over the provided `replace_paths` in reverse order + (from most specific to least specific) and replaces the first matching base + path in the given `path` with the corresponding environment variable name. + + Examples + -------- + >>> path = '/home/user/geoproc/geoips_packages/module/file.py' + >>> base_paths = { + ... 'GEOIPS_PACKAGES_DIR': '/home/user/geoproc/geoips_packages', + ... 'GEOIPS_BASEDIR': '/home/user/geoproc' + ... } + >>> replace_geoips_paths(path, base_paths=base_paths) + '$GEOIPS_PACKAGES_DIR/module/file.py' """ # Allow multiple sets of base_path replacements - from geoips.filenames.base_paths import PATHS as geoips_gpaths if base_paths is None: - base_paths = [geoips_gpaths] + base_paths = geoips_paths + + if replace_paths is None: + replace_env_vars = [ + "$GEOIPS_OUTDIRS", + "$GEOIPS_PACKAGES_DIR", + "$GEOIPS_TESTDATA_DIR", + "$GEOIPS_DEPENDENCIES_DIR", + "$GEOIPS_BASEDIR", + ] + + paths_to_be_replaced = [Path(os.path.expandvars(p)) for p in replace_env_vars] + ordered_path_envvar_dict = { + replace_env_vars[paths_to_be_replaced.index(replace_path)]: replace_path + for replace_path in order_paths_from_least_to_most_specific( + paths_to_be_replaced + ) + } # Replace with specified file system -> URL mapping - for paths in base_paths: - for key in paths.keys(): - if f"{key}_URL" in paths: - fname = fname.replace(paths[key], paths[f"{key}_URL"]) + # for paths in base_paths: + # for key in paths.keys(): + # if f"{key}_URL" in paths: + # fname = fname.replace(paths[key], paths[f"{key}_URL"]) + + path = Path(os.path.expandvars(path)) # Replace full paths with environment variables - if replace_paths is None: - replace_paths = [ - "GEOIPS_OUTDIRS", - "GEOIPS_PACKAGES_DIR", - "GEOIPS_TESTDATA_DIR", - "GEOIPS_DEPENDENCIES_DIR", - "GEOIPS_BASEDIR", - ] - for replace_path in replace_paths: - for paths in base_paths: - if replace_path in paths: - if curly_braces: - fname = fname.replace(paths[replace_path], f"${{{replace_path}}}") - else: - fname = fname.replace(paths[replace_path], f"${replace_path}") - return fname + for env_var, replace_path in ordered_path_envvar_dict.items(): + if replace_path in path.parents: + env_var = env_var.replace("$", "") + return str(path).replace( + str(replace_path), + f"${{{env_var}}}" if curly_braces else f"${env_var}", + ) + return str(path) def get_required_geoips_xarray_attrs(): diff --git a/geoips/interfaces/module_based/output_checkers.py b/geoips/interfaces/module_based/output_checkers.py index 18f6142b8..a884225be 100644 --- a/geoips/interfaces/module_based/output_checkers.py +++ b/geoips/interfaces/module_based/output_checkers.py @@ -1012,7 +1012,7 @@ def identify_checker(self, filename): return checker_name def get_plugin(self, name): - """Return the output checker plugin corresponding to checker_name.""" + """Get the output checker plugin corresponding to checker_name.""" plug = super().get_plugin(name) if self.valid_plugin(plug): return plug diff --git a/geoips/plugins/modules/colormappers/winds/wind_radii_transitions.py b/geoips/plugins/modules/colormappers/winds/wind_radii_transitions.py index 20aa2ad82..e455de3c1 100644 --- a/geoips/plugins/modules/colormappers/winds/wind_radii_transitions.py +++ b/geoips/plugins/modules/colormappers/winds/wind_radii_transitions.py @@ -37,36 +37,36 @@ def call(data_range=[0, 200]): min_wind_speed = data_range[0] max_wind_speed = data_range[1] transition_vals = [ - (min_wind_speed, 34), + (min_wind_speed, 12), + (12, 25), + (25, 34), (34, 50), (50, 64), (64, 80), - # (64, 72), - # (72, 80), (80, 100), (100, 120), - (120, 150), - (150, max_wind_speed), + (120, max_wind_speed), ] transition_colors = [ - ("lightblue", "blue"), + ("white", "#739FE1"), + ("#3f82ff", "blue"), + ("#94d9a7", "#317E0B"), ("yellow", "orange"), - ("red", "red"), - # ('thistle', 'thistle'), - # ('firebrick', 'firebrick'), - # ('fuchsia', 'fuchsia'), - # ('mediumvioletred', 'mediumvioletred'), - ("rebeccapurple", "rebeccapurple"), - # ('purple', 'rebeccapurple'), - # ('rebeccapurple', 'rebeccapurple'), - # ('mediumvioletred', 'mediumvioletred'), - ("palevioletred", "palevioletred"), - ("silver", "silver"), - ("gray", "gray"), - ("dimgray", "dimgray"), + ("#ff7878", "#C90A0A"), + ("#C285F6", "rebeccapurple"), + ("#fcb4cc", "palevioletred"), + # Grays + # ("#BCB8B8", "#999898"), + # ("#999898", "#808080"), + # ("#808080", "#737373"), + # ("#737373", "#5a5a5a"), + # ("#5a5a5a", "#3d3d3d"), + ("#BCB8B8", "dimgray"), + ("darkslategray", "black"), ] - ticks = [xx[0] for xx in transition_vals] + # ticks = [xx[0] for xx in transition_vals] + ticks = [0, 12, 25, 34, 50, 64, 80, 100, 120, 150, 200] min_wind_speed = transition_vals[0][0] max_wind_speed = transition_vals[-1][1] @@ -104,6 +104,7 @@ def call(data_range=[0, 200]): "boundaries": mpl_boundaries, "cbar_spacing": cbar_spacing, "colorbar": True, + "cbar_full_width": True, } # return cbar, min_wind_speed, max_wind_speed diff --git a/geoips/plugins/modules/colormappers/winds/wind_radii_transitions_legacy.py b/geoips/plugins/modules/colormappers/winds/wind_radii_transitions_legacy.py new file mode 100644 index 000000000..ea92f1aac --- /dev/null +++ b/geoips/plugins/modules/colormappers/winds/wind_radii_transitions_legacy.py @@ -0,0 +1,110 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + +"""Module containing wind speed colormap with transitions at 34, 50, 64, and 80.""" + +import logging + +LOG = logging.getLogger(__name__) + +interface = "colormappers" +family = "matplotlib" +name = "wind_radii_transitions_legacy" + + +def call(data_range=[0, 200]): + """Generate appropriate matplotlib colors for plotting standard wind speeds. + + wind_radii_transitions contains hard coded transition values for different + colors, in order to have consistent imagery across all sensors / products. + + Parameters + ---------- + data_range : list of float, default=[0, 200] + * Min and max value for colormap. + * Ensure the data range matches the range of the algorithm specified for + use with this colormap + * This colormap MUST include 0 and 200 + + Returns + ------- + mpl_colors_info : dict + Dictionary of matplotlib plotting parameters, to ensure consistent + image output + """ + from geoips.image_utils.colormap_utils import create_linear_segmented_colormap + + min_wind_speed = data_range[0] + max_wind_speed = data_range[1] + transition_vals = [ + (min_wind_speed, 34), + (34, 50), + (50, 64), + (64, 80), + # (64, 72), + # (72, 80), + (80, 100), + (100, 120), + (120, 150), + (150, max_wind_speed), + ] + transition_colors = [ + ("lightblue", "blue"), + ("yellow", "orange"), + ("red", "red"), + # ('thistle', 'thistle'), + # ('firebrick', 'firebrick'), + # ('fuchsia', 'fuchsia'), + # ('mediumvioletred', 'mediumvioletred'), + ("rebeccapurple", "rebeccapurple"), + # ('purple', 'rebeccapurple'), + # ('rebeccapurple', 'rebeccapurple'), + # ('mediumvioletred', 'mediumvioletred'), + ("palevioletred", "palevioletred"), + ("silver", "silver"), + ("gray", "gray"), + ("dimgray", "dimgray"), + ] + + ticks = [xx[0] for xx in transition_vals] + + min_wind_speed = transition_vals[0][0] + max_wind_speed = transition_vals[-1][1] + + LOG.info("Setting cmap") + mpl_cmap = create_linear_segmented_colormap( + "windspeed_cmap", + min_wind_speed, + max_wind_speed, + transition_vals, + transition_colors, + ) + + LOG.info("Setting norm") + from matplotlib.colors import Normalize + + mpl_norm = Normalize(vmin=min_wind_speed, vmax=max_wind_speed) + + cbar_label = "Surface Wind (knots)" + + # Must be uniform or proportional, None not valid for Python 3 + cbar_spacing = "proportional" + mpl_tick_labels = None + mpl_boundaries = None + + # from geoips.image_utils.mpl_utils import create_colorbar + # only create colorbar for final imagery + # cbar = create_colorbar(fig, mpl_cmap, mpl_norm, ticks, cbar_label=cbar_label) + mpl_colors_info = { + "cmap": mpl_cmap, + "norm": mpl_norm, + "cbar_ticks": ticks, + "cbar_tick_labels": mpl_tick_labels, + "cbar_label": cbar_label, + "boundaries": mpl_boundaries, + "cbar_spacing": cbar_spacing, + "colorbar": True, + } + + # return cbar, min_wind_speed, max_wind_speed + return mpl_colors_info diff --git a/geoips/plugins/modules/interpolators/utils/interp_scipy.py b/geoips/plugins/modules/interpolators/utils/interp_scipy.py index 7228332d5..7f4aa2c12 100644 --- a/geoips/plugins/modules/interpolators/utils/interp_scipy.py +++ b/geoips/plugins/modules/interpolators/utils/interp_scipy.py @@ -10,6 +10,10 @@ import scipy import numpy +# imports for alphashape-based masking +import alphashape +from skimage.measure import points_in_poly + LOG = logging.getLogger(__name__) # interface = None indicates to the GeoIPS interfaces that this is not a valid @@ -145,12 +149,36 @@ def interp_griddata( interp_data = scipy.interpolate.griddata( (data_lats, data_lons), data_array, (gridlats, gridlons), method=method ) + list_of_lons = numpy.reshape(gridlons, -1) + list_of_lats = numpy.reshape(gridlats, -1) + # Free up memory ?? gridlons = 1 gridlats = 1 + # Zip data_lons and data_lats and compute alphashape + datapts_2d = list(zip(data_lons.data, data_lats.data)) + returned_polygons = alphashape.alphashape(datapts_2d, 1.0) + # If multiple polygons are returned, choose the one with the largest area + if returned_polygons.geom_type == "MultiPolygon": + alpha_shape = max(returned_polygons.geoms, key=lambda a: a.area) + elif returned_polygons.geom_type == "Polygon": + alpha_shape = returned_polygons + interp_data = numpy.ma.masked_invalid(interp_data) interp_data = numpy.ma.masked_less(interp_data, data_array.min()) interp_data = numpy.ma.masked_greater(interp_data, data_array.max()) + input_shape_interp_data = interp_data.shape + + zipped_lons_lats = numpy.asarray(list(zip(list_of_lons, list_of_lats))) + mask_exterior_coords_xy = numpy.asarray(list(zip(*alpha_shape.exterior.coords.xy))) + inside_hull_mask = numpy.reshape( + points_in_poly(zipped_lons_lats, mask_exterior_coords_xy), + input_shape_interp_data, + ) + + interp_data = numpy.ma.masked_where( + numpy.logical_not(inside_hull_mask), interp_data + ) return interp_data diff --git a/geoips/plugins/modules/output_checkers/image.py b/geoips/plugins/modules/output_checkers/image.py index fe243b933..2d4b7e432 100644 --- a/geoips/plugins/modules/output_checkers/image.py +++ b/geoips/plugins/modules/output_checkers/image.py @@ -6,6 +6,8 @@ import logging from os.path import splitext +from geoips.filenames.base_paths import PATHS as gpaths + LOG = logging.getLogger(__name__) interface = "output_checkers" @@ -84,7 +86,12 @@ def correct_file_format(fname): return False -def outputs_match(plugin, output_product, compare_product, threshold): +def outputs_match( + plugin, + output_product, + compare_product, + threshold=gpaths["OUTPUT_CHECKER_THRESHOLD_IMAGE"], +): """Use PIL and numpy to compare two images. Parameters @@ -95,7 +102,7 @@ def outputs_match(plugin, output_product, compare_product, threshold): Current output product compare_product : str Path to comparison product - threshold: float + threshold: float, default=0.05 Threshold for the image comparison. Argument to pixelmatch. Between 0 and 1, with 0 the most strict comparison, and 1 the most lenient. @@ -234,7 +241,12 @@ def outputs_match(plugin, output_product, compare_product, threshold): return True -def call(plugin, compare_path, output_products, threshold=0.05): +def call( + plugin, + compare_path, + output_products, + threshold=gpaths["OUTPUT_CHECKER_THRESHOLD_IMAGE"], +): """Compare the "correct" imagery found the list of current output_products. Compares files produced in the current processing run with the list of diff --git a/geoips/plugins/modules/output_formatters/histogram_csv.py b/geoips/plugins/modules/output_formatters/histogram_csv.py new file mode 100644 index 000000000..bd1d3a4cd --- /dev/null +++ b/geoips/plugins/modules/output_formatters/histogram_csv.py @@ -0,0 +1,62 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + +"""Produce histogram from the given dataset with specified bin size.""" +import logging +import numpy + +LOG = logging.getLogger(__name__) + +interface = "output_formatters" +family = "xarray_data" +name = "histogram_csv" + + +def call( + xarray_obj, product_names, output_fnames, min_val=None, max_val=None, num_bins=501 +): + """Produce histogram. + + Parameters + ---------- + xarray_obj: xarray Dataset + + Returns + ------- + xarray.Dataset + Resulting histogram + """ + import xarray + + prod_xarray = xarray.Dataset() + + from geoips.geoips_utils import copy_standard_metadata + from geoips.filenames.base_paths import make_dirs + from os.path import dirname + + copy_standard_metadata(xarray_obj, prod_xarray) + for product_name in product_names: + prod_xarray[product_name] = xarray_obj[product_name] + + if product_name == "latitude" or product_name == "longitude": + continue + xda = xarray_obj[product_name] + hist = numpy.histogram(xda, numpy.linspace(min_val, max_val, num_bins + 1)) + prod_xarray[f"{product_name}_histogram"] = xarray.DataArray( + hist[0], dims=("dim_2") + ) + prod_xarray["bins"] = xarray.DataArray(hist[1], dims=("dim_3")) + for ncdf_fname in output_fnames: + make_dirs(dirname(ncdf_fname)) + with open(ncdf_fname, "w") as fobj: + prod = prod_xarray[f"{product_name}_histogram"].to_numpy() + fobj.write(",".join([str(val) for val in list(prod)])) + LOG.interactive(f"Writing {ncdf_fname}") + + # from geoips.plugins.modules.output_formatters.netcdf_xarray import ( + # write_xarray_netcdf, + # ) + + # for ncdf_fname in output_fnames: + # write_xarray_netcdf(prod_xarray, ncdf_fname) + return output_fnames diff --git a/geoips/plugins/modules/output_formatters/netcdf_geoips.py b/geoips/plugins/modules/output_formatters/netcdf_geoips.py index 2e766eacc..4154cf8fd 100644 --- a/geoips/plugins/modules/output_formatters/netcdf_geoips.py +++ b/geoips/plugins/modules/output_formatters/netcdf_geoips.py @@ -11,7 +11,7 @@ name = "netcdf_geoips" -def call(xarray_obj, product_names, output_fnames): +def call(xarray_obj, product_names, output_fnames, clobber=False): """Write GeoIPS style NetCDF to disk.""" import xarray @@ -28,5 +28,5 @@ def call(xarray_obj, product_names, output_fnames): ) for ncdf_fname in output_fnames: - write_xarray_netcdf(prod_xarray, ncdf_fname) + write_xarray_netcdf(prod_xarray, ncdf_fname, clobber=clobber) return output_fnames diff --git a/geoips/plugins/modules/output_formatters/netcdf_xarray.py b/geoips/plugins/modules/output_formatters/netcdf_xarray.py index 7b1141993..a4007ae8d 100644 --- a/geoips/plugins/modules/output_formatters/netcdf_xarray.py +++ b/geoips/plugins/modules/output_formatters/netcdf_xarray.py @@ -12,7 +12,7 @@ name = "netcdf_xarray" -def call(xarray_obj, product_names, output_fnames): +def call(xarray_obj, product_names, output_fnames, clobber=False): """Write xarray-based NetCDF outputs to disk.""" for ncdf_fname in output_fnames: write_xarray_netcdf(xarray_obj, ncdf_fname) diff --git a/geoips/plugins/modules/procflows/config_based.py b/geoips/plugins/modules/procflows/config_based.py index 255c9b732..a8297c1bf 100644 --- a/geoips/plugins/modules/procflows/config_based.py +++ b/geoips/plugins/modules/procflows/config_based.py @@ -756,6 +756,23 @@ def get_area_defs_from_available_sectors( if sector_dict.get("trackfiles"): sector_dict["tcdb"] = False + # Check if sector_list specified under YAML output config file is a list or a + # dictionary. If sector_list is a list, static sectors are enabled for all + # platforms that use the output config YAML. If sector_list is a dictionary, + # each key is a platform name that holds a list of static sectors to be + # processed for said platform. If sector_list is a dictionary, and the platform + # name is not a key, warning is raised and sector_list is set as an empty list. + if sector_dict.get("sector_list") and isinstance( + sector_dict.get("sector_list"), dict + ): + try: + sector_dict["sector_list"] = sector_dict["sector_list"][ + xobjs["METADATA"].platform_name + ] + except KeyError as resp: + LOG.warning("%s MISSING PLATFORM NAME", resp) + sector_dict["sector_list"] = [] + # This is the standard "get_area_defs_from_command_line_args", YAML config # specified sector information matches the command line specified sector # information diff --git a/geoips/plugins/modules/procflows/single_source.py b/geoips/plugins/modules/procflows/single_source.py index f695a8e9b..77c2da341 100644 --- a/geoips/plugins/modules/procflows/single_source.py +++ b/geoips/plugins/modules/procflows/single_source.py @@ -609,7 +609,7 @@ def apply_interp_after_alg( interp_plugin, area_def, alg_xarray, - alg_xarray, + xarray.Dataset(), interp_args, processed_xarrays, ) diff --git a/geoips/plugins/modules/readers/cygnss_netcdf.py b/geoips/plugins/modules/readers/cygnss_netcdf.py new file mode 100644 index 000000000..53611081a --- /dev/null +++ b/geoips/plugins/modules/readers/cygnss_netcdf.py @@ -0,0 +1,114 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + +"""Read derived surface winds from CYGNSS netcdf data.""" + +import glob +import logging +import xarray + +from geoips.xarray_utils.time import get_min_from_xarray_time, get_max_from_xarray_time + +LOG = logging.getLogger(__name__) + +MS_TO_KTS = 1.94384 +DEG_TO_KM = 111.321 + +interface = "readers" +family = "standard" +name = "cygnss_netcdf" + + +def call(fnames, metadata_only=False, chans=None, area_def=None, self_register=False): + """Read CYGNSS derived winds from netcdf data. + + Parameters + ---------- + fnames : list + * List of strings, full paths to files + metadata_only : bool, default=False + * Return before actually reading data if True + chans : list of str, default=None + * List of desired channels (skip unneeded variables as needed). + * Include all channels if None. + area_def : pyresample.AreaDefinition, default=None + * NOT YET IMPLEMENTED + * Specify region to read + * Read all data if None. + self_register : str or bool, default=False + * NOT YET IMPLEMENTED + * register all data to the specified dataset id (as specified in the + return dictionary keys). + * Read multiple resolutions of data if False. + + Returns + ------- + dict of xarray.Datasets + * dictionary of xarray.Dataset objects with required Variables and + Attributes. + * Dictionary keys can be any descriptive dataset ids. + + See Also + -------- + :ref:`xarray_standards` + Additional information regarding required attributes and variables + for GeoIPS-formatted xarray Datasets. + """ + out_dict = {} + + # Single file is full day of data, so only need one. + fname = fnames[0] + + wind_xarray = xarray.open_dataset(str(fname)) + wind_xarray = wind_xarray.rename( + {"lat": "latitude", "lon": "longitude", "sample_time": "time"} + ) + wind_xarray.attrs["source_name"] = "cygnss" + wind_xarray.attrs["platform_name"] = "cygnss" + wind_xarray.attrs["data_provider"] = "noaa-nesdis-star-socd-oswt" + wind_xarray.attrs["source_file_names"] = fnames + wind_xarray.attrs["interpolation_radius_of_influence"] = 20000 + wind_xarray.attrs["sample_distance_km"] = DEG_TO_KM / 4 + + wind_xarray.attrs["start_datetime"] = get_min_from_xarray_time(wind_xarray, "time") + wind_xarray.attrs["end_datetime"] = get_max_from_xarray_time(wind_xarray, "time") + + out_dict["METADATA"] = wind_xarray[[]] + if metadata_only: + return out_dict + + if chans: + if "wind_speed_kts" in chans: + wind_xarray["wind_speed_kts"] = wind_xarray["wind_speed"] * MS_TO_KTS + wind_xarray["wind_speed_kts"].attrs["units"] = "kts" + + ds_var_list = list(wind_xarray.variables.keys()) + + if chans: + for chan in chans: + if chan not in ds_var_list: + raise ValueError(f"Variable {chan} not found in dataset.") + + wind_xarray = wind_xarray[chans] + + out_dict["CYGNSS"] = wind_xarray + + return out_dict + + +# Unit test functions +def get_test_files(test_data_dir): + """Generate testing xarray from test data.""" + filepath = test_data_dir + "/test_data_cygnss/data/*.nc" + filelist = glob.glob(filepath) + tmp_xr = call(filelist) + if len(filelist) == 0: + raise NameError("No files found") + return tmp_xr + + +def get_test_parameters(): + """Generate test data key for unit testing.""" + return [ + {"data_key": "CYGNSS", "data_var": "wind_speed", "mean": 7.0251474}, + ] diff --git a/geoips/plugins/modules/readers/scat_knmi_winds_netcdf.py b/geoips/plugins/modules/readers/scat_knmi_winds_netcdf.py index 3a5143555..bb168bba7 100644 --- a/geoips/plugins/modules/readers/scat_knmi_winds_netcdf.py +++ b/geoips/plugins/modules/readers/scat_knmi_winds_netcdf.py @@ -43,6 +43,9 @@ def read_knmi_data(wind_xarray): elif wind_xarray.source == "ScatSat-1 OSCAT": geoips_metadata["source_name"] = "oscat" geoips_metadata["platform_name"] = "scatsat-1" + elif wind_xarray.source == "Oceansat-3 OSCAT": + geoips_metadata["source_name"] = "oscat" + geoips_metadata["platform_name"] = "oceansat-3" elif wind_xarray.source == "HY-2D HSCAT": geoips_metadata["source_name"] = "hscat" geoips_metadata["platform_name"] = "hy-2d" diff --git a/geoips/plugins/modules/readers/viirs_sdr_hdf5.py b/geoips/plugins/modules/readers/viirs_sdr_hdf5.py new file mode 100755 index 000000000..d009d99f5 --- /dev/null +++ b/geoips/plugins/modules/readers/viirs_sdr_hdf5.py @@ -0,0 +1,317 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + +"""VIIRS SDR Satpy reader. + +This VIIRS reader is designed for reading the NPP/JPSS SDR HDF5 files. +The input files are produced by CSPP Polar (CSPP RDR pipeline), +and the read by satpy. + +V1.1.0: NRL-Monterey, Aug. 2024 + +""" + +# Python Standard Libraries +import logging +import os + +# Installed Libraries +import xarray as xr +import numpy as np +import h5py +from pandas import date_range +from pykdtree.kdtree import KDTree + +# If this reader is not installed on the system, don't fail altogether, just skip this +# import. This reader will not work if the import fails, and the package will have to be +# installed to process data of this type. +LOG = logging.getLogger(__name__) + +interface = "readers" +family = "standard" +name = "viirs_sdr_hdf5" + +try: + import satpy +except ImportError: + LOG.info("Failed import satpy. If you need it, install it.") + +VARLIST = { + "DNB": ["DNB"], + "IMG": ["I04", "I05"], + "IMG-Vis": ["I01", "I02", "I03"], + "MOD": ["M07", "M08", "M10", "M11", "M12", "M13", "M14", "M15", "M16"], + "MOD-Vis": ["M01", "M02", "M03", "M04", "M05", "M06", "M09"], +} + + +def bowtie_correction(band, lat, lon): + """Correct input data for the instrument bowtie effect. + + Correction derived from: Remote Sens. 2016, 8, 79. + """ + # Unfold + ord_lat = np.sort(lat, axis=0) + unfold_idx = np.argsort(lat, axis=0) + # no_shift_flag = np.argsort(ord_lat, axis=0) == unfold_idx + + rad_fold = np.take_along_axis(band, unfold_idx, axis=0) + sort_lon = np.take_along_axis(lon, unfold_idx, axis=0) + + if np.all(np.isnan(rad_fold)): + LOG.debug("All nan band, no bowtie correction") + return rad_fold, ord_lat, sort_lon + + # Adjust lon, not used for satpy + # Only need for manual read with overlapping granuales + # ord_lon = np.empty(lon.shape) + # xi = np.arange(sort_lon.shape[0]) + + # for x in range(lon.shape[1]): + # 0 shift xi values + # xo = xi[no_shift_flag[:, x]] + + # if all(no_shift_flag[:, x]): + # if there was no shift in the column + # ord_lon[:, x] = lon[:, x] + # continue + # elif not any(no_shift_flag[:, x]): + # if the whole column was shifted (should be rare) + # ord_lon[:, x] = sort_lon[:, x] + # continue + + # longitude values that were not shifted + # noshift_lon = sort_lon[xo,x] + # noshift_lon = sort_lon[:, x][no_shift_flag[:, x]] + # replace only values that were shifted + # nsf = no_shift_flag[:, x] + + # ord_lon[nsf, x] = noshift_lon + # ord_lon[~nsf, x] = np.interp(xi, xo, noshift_lon)[~nsf] + + # Resample + point_mask = np.isnan(rad_fold) + + good_points = np.dstack((ord_lat[~point_mask], sort_lon[~point_mask]))[0] + bad_points = np.dstack((ord_lat[point_mask], sort_lon[point_mask]))[0] + + res_band = rad_fold.copy() + good_rad = rad_fold[~point_mask] + rad_idx = np.indices(rad_fold.shape) + ridx, ridy = rad_idx[0][point_mask], rad_idx[1][point_mask] + os.environ["OMP_NUM_THREADS"] = "64" + + kd_tree = KDTree(good_points) + # print("Querying") + dist, idx = kd_tree.query(bad_points, k=4) # ,workers=4) + + for i in range(bad_points.shape[0]): + xi, yi = ridx[i], ridy[i] + + if np.any(dist[i] == 0): + # weight the zero to a small value + weight = np.where(dist[i] == 0, 1e-6, dist[i]) + res_band[xi, yi] = np.average(good_rad[idx[i]], weights=1 / weight) + continue + + res_band[xi, yi] = np.average(good_rad[idx[i]], weights=1 / dist[i]) + + return res_band, ord_lat.astype(np.float64), sort_lon.astype(np.float64) + + +def call(fnames, metadata_only=False, chans=None, area_def=None, self_register=False): + """Read VIIRS SDR hdf5 data products. + + Parameters + ---------- + fnames : list + * List of strings, full paths to files + metadata_only : bool, default=False + * Return before actually reading data if True + chans : list of str, default=None + * List of desired channels (skip unneeded variables as needed). + * Include all channels if None. + area_def : pyresample.AreaDefinition, default=None + * NOT YET IMPLEMENTED + * Specify region to read + * Read all data if None. + self_register : str or bool, default=False + * NOT YET IMPLEMENTED + * register all data to the specified dataset id (as specified in the + return dictionary keys). + * Read multiple resolutions of data if False. + + Returns + ------- + dict of xarray.Datasets + * dictionary of xarray.Dataset objects with required Variables and + Attributes. + * Dictionary keys can be any descriptive dataset ids. + * Conforms to geoips xarray standards, see more in geoips documentation. + """ + # print("Reading") + tmp_scn = satpy.Scene(reader="viirs_sdr", filenames=fnames) + scn_start, scn_end = tmp_scn.start_time, tmp_scn.end_time + base_fnames = list(map(os.path.basename, fnames)) + + full_xr = {} + if metadata_only: + # average resolution + # sensor, plat name + tmp_scn.load([tmp_scn.available_dataset_names()[0]]) + tmp_attrs = tmp_scn[tmp_scn.available_dataset_names()[0]].attrs + tmp_xr = xr.Dataset( + attrs={ + "source_file_name": base_fnames[0], + "start_datetime": scn_start, + "end_datetime": scn_end, + "source_name": tmp_attrs["sensor"], + "platform_name": tmp_attrs["platform_name"], + "data_provider": "NOAA", + "sample_distance_km": 1, + "interpolation_radius_of_influence": 1000, # guess! + } + ) + tmp_dict = {"METADATA": tmp_xr} + return tmp_dict + + # trim VARLIST based on channels requested + if chans: + tmp_vl = VARLIST.copy() + for key, val in tmp_vl.items(): + km = [c[:3] in val for c in chans] + if not any(km): + VARLIST.pop(key) + else: + matches = [s for s in val for c in chans if s in c] + VARLIST[key] = matches + + # could opimize more + tmp_coor = {} + for var in VARLIST: + tmp_dask = {} + dataset_ids = [ + idx + for idx in tmp_scn.available_dataset_ids() + if idx["name"] in VARLIST[var] + ] + if len(dataset_ids) == 0: + # print("No datasets found for {}.".format(VARLIST[var])) + continue + + for d in dataset_ids: + # print("Loading {}".format(d)) + tmp_scn.load([d]) + full_key = tmp_scn[d].attrs["name"] + tmp_scn[d].attrs[ + "calibration" + ].capitalize()[:3].replace("Bri", "BT") + + tmp_ma = tmp_scn[d].to_masked_array().data + + # + tmp_scn.load([d]) + + lat = tmp_scn[d].area.lats.to_masked_array().data + lon = tmp_scn[d].area.lons.to_masked_array().data + + # bowtie correction + band_data, band_lat, band_lon = bowtie_correction(tmp_ma, lat, lon) + + tmp_dask |= {full_key: (("dim_0", "dim_1"), band_data)} + + # coordinates + tmp_coor["latitude"] = ( + ("dim_0", "dim_1"), + band_lat, + ) + tmp_coor["longitude"] = ( + ("dim_0", "dim_1"), + band_lon, + ) + # print("Setting cal vals") + # sample time to the proper shape (N*48), while lat/lon are () + time_range = date_range( + start=scn_start, end=scn_end, periods=tmp_coor["latitude"][1].shape[0] + ).values + interp_time = np.tile(time_range, (tmp_coor["latitude"][1].shape[1], 1)).T + tmp_coor["time"] = (("dim_0", "dim_1"), interp_time) + # # print(tmp_coor["latitude"][1].shape) + # raise + + tmp_attrs = tmp_scn[VARLIST[var][0]].attrs + + cal_params = [ + "satellite_azimuth_angle", + "satellite_zenith_angle", + "solar_azimuth_angle", + "solar_zenith_angle", + ] + + if var == "DNB": + cal_params = [ + "dnb_lunar_azimuth_angle", + "dnb_lunar_zenith_angle", + "dnb_satellite_azimuth_angle", + "dnb_satellite_zenith_angle", + "dnb_solar_azimuth_angle", + "dnb_solar_zenith_angle", + ] + + tmp_scn.load(cal_params) + tmp_cal_params = { + i.removeprefix("dnb_"): (("dim_0", "dim_1"), tmp_scn[i].to_masked_array()) + for i in cal_params + } + + if var == "DNB": + try: + from lunarref.lib.liblunarref import lunarref + + # tmp_scn.load(["dnb_moon_illumination_fraction"]) + # this results in the wrong value.. + # np.arccos((tmp_scn["dnb_moon_illumination_fraction"].data/50)-1) + + dnb_geofile = [i for i in fnames if "GDNBO" in os.path.basename(i)][0] + h5_dnb = h5py.File(dnb_geofile) + phase_ang = h5_dnb["All_Data/VIIRS-DNB-GEO_All/MoonPhaseAngle"][...] + + lunarref_data = lunarref( + tmp_dask["DNBRad"][1], + tmp_cal_params["solar_zenith_angle"][1], + tmp_cal_params["lunar_zenith_angle"][1], + scn_start.strftime("%Y%m%d%H"), + scn_start.strftime("%M"), + phase_ang, + ) + lunarref_data = np.ma.masked_less_equal(lunarref_data, -999, copy=False) + tmp_dask |= {"DNBRef": (("dim_0", "dim_1"), lunarref_data)} + except ImportError: + LOG.info("Failed lunarref in viirs reader. If you need it, build it") + + # problem with sat_za/az values being too high, need to downsample + # print("Building xarray") + obs_xr = xr.Dataset(data_vars=tmp_dask) + coor_xr = xr.Dataset(data_vars=tmp_coor) + cal_xr = xr.Dataset(data_vars=tmp_cal_params) + + try: + tmp_xr = xr.merge([obs_xr, coor_xr, cal_xr]) + except ValueError: + # downsample for certain bands + tmp_xr = xr.merge([obs_xr, coor_xr]) + + tmp_xr.attrs = { + "source_file_name": base_fnames, + "start_datetime": tmp_scn.start_time, + "end_datetime": tmp_scn.end_time, + "source_name": tmp_attrs["sensor"], + "platform_name": tmp_attrs["platform_name"], + "data_provider": "NOAA", + "sample_distance_km": tmp_attrs["resolution"] / 1e3, + "interpolation_radius_of_influence": 1000, + } + + full_xr |= {var: tmp_xr} + full_xr["METADATA"] = xr.Dataset(attrs=tmp_xr.attrs) + + return full_xr diff --git a/geoips/plugins/yaml/product_defaults/pmw_89/89pct.yaml b/geoips/plugins/yaml/product_defaults/pmw_89/89pct.yaml index 23f193bb5..d40e3161e 100644 --- a/geoips/plugins/yaml/product_defaults/pmw_89/89pct.yaml +++ b/geoips/plugins/yaml/product_defaults/pmw_89/89pct.yaml @@ -18,7 +18,7 @@ spec: plugin: name: pmw_89pct arguments: - data_range: [105, 280] + data_range: [105.0, 280.0] interpolator: plugin: name: interp_gauss diff --git a/geoips/plugins/yaml/products/ascat.yaml b/geoips/plugins/yaml/products/ascat.yaml index 494f9eaa7..f7dc41254 100644 --- a/geoips/plugins/yaml/products/ascat.yaml +++ b/geoips/plugins/yaml/products/ascat.yaml @@ -46,7 +46,8 @@ spec: - name: wind-ambiguities source_names: [ascat] docstring: | - The wind-ambiguities product_defaults for ascat product. + Scatterometer data that shows and abstract of possible wind + direction. Usually have four cases so the first two shown are most likely. product_defaults: wind-ambiguities spec: variables: diff --git a/geoips/plugins/yaml/products/cygnss.yaml b/geoips/plugins/yaml/products/cygnss.yaml new file mode 100644 index 000000000..2656ae61d --- /dev/null +++ b/geoips/plugins/yaml/products/cygnss.yaml @@ -0,0 +1,15 @@ +interface: products +family: list +name: cygnss +docstring: | + The cygnss products configuration, which produces Windspeed products, Windbarbs products, + Wind-ambiguity products, and Unsectored/Sectored products. +spec: + products: + - name: windspeed + source_names: [cygnss] + docstring: | + The windspeed product_defaults for cygnss product. + product_defaults: windspeed + spec: + variables: ["wind_speed_kts"] \ No newline at end of file diff --git a/geoips/plugins/yaml/products/oscat.yaml b/geoips/plugins/yaml/products/oscat.yaml index b8124ff66..c4e4de3b6 100644 --- a/geoips/plugins/yaml/products/oscat.yaml +++ b/geoips/plugins/yaml/products/oscat.yaml @@ -31,3 +31,17 @@ spec: "wind_dir_deg_ambiguity_met", "rain_flag_ambiguity", ] + - name: unsectored + source_names: [oscat] + docstring: | + The unsectored product_defaults for oscat product. + product_defaults: unsectored + spec: + variables: ["wind_speed_kts", "wind_dir_deg_met"] + - name: sectored + source_names: [oscat] + docstring: | + The sectored product_defaults for oscat product. + product_defaults: sectored + spec: + variables: ["wind_speed_kts", "wind_dir_deg_met"] diff --git a/geoips/plugins/yaml/products/smos-spd.yaml b/geoips/plugins/yaml/products/smos-spd.yaml index b763fa5da..2c5443510 100644 --- a/geoips/plugins/yaml/products/smos-spd.yaml +++ b/geoips/plugins/yaml/products/smos-spd.yaml @@ -2,6 +2,8 @@ interface: products family: list name: smos-spd docstring: | + Soil Moisture sensor that “smart people can use for windspeed” (Sampson, Meteorologist). + Wavelength (3 cm) is long enough to penetrate heavy rains but shows low resolution (40 km). The smos-spd product_inputs configuration, which produces windspeed, unsectored, and sectored products. spec: products: diff --git a/geoips/sector_utils/tc_tracks_database.py b/geoips/sector_utils/tc_tracks_database.py index 413101a32..c5bf99db6 100644 --- a/geoips/sector_utils/tc_tracks_database.py +++ b/geoips/sector_utils/tc_tracks_database.py @@ -116,13 +116,17 @@ def update_fields(tc_trackfilename, cc, conn, process=False): # Check if timestamp on file is newer than timestamp in database - # if not, just return and don't do anything. if data: - database_timestamp = datetime.strptime( - cc.execute( - "SELECT last_updated from tc_trackfiles WHERE filename = ?", - (tc_trackfilename,), - ).fetchone()[0], - "%Y-%m-%d %H:%M:%S.%f", - ) + try: + database_timestamp = datetime.strptime( + cc.execute( + "SELECT last_updated from tc_trackfiles WHERE filename = ?", + (tc_trackfilename,), + ).fetchone()[0], + "%Y-%m-%d %H:%M:%S.%f", + ) + except ValueError as resp: + LOG.exception(f"Failed on {tc_trackfilename}") + raise (ValueError(f"FAILED ON {tc_trackfilename}: {resp}")) if file_timestamp < database_timestamp: LOG.info("") LOG.interactive( diff --git a/geoips/utils/memusg.py b/geoips/utils/memusg.py index 42f9eb606..942c01eed 100644 --- a/geoips/utils/memusg.py +++ b/geoips/utils/memusg.py @@ -3,6 +3,7 @@ """Utilities for tracking and monitoring memory and resource usage.""" # Python standard Libraries +# Python standard Libraries import logging import socket import os diff --git a/pyproject.toml b/pyproject.toml index 6da219a9c..609d77f4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,10 +56,12 @@ requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"] build-backend = "poetry_dynamic_versioning.backend" # object used to perform the build process [tool.poetry.dependencies] # must download to run +# NOTE this setuptools dependency should be removed altogether once meson fortran builds are working +setuptools = "<70" # Required for any fortran builds until meson working python = ">=3.10.0" # mandatory to declare the required python version matplotlib = ">=3.7.0" # Base requirement works, version specific to test outputs -netcdf4 = "*" # Base requirement -numpy = "<2.0.0" # Base requirement +netcdf4 = "<1.7.0" # Base requirement, netcdf 1.7.0 causes seg fault in ABI reader +numpy = "<2.0" # Base requirement numpy 1.26.4 still works, 2.0 breaks numexpr pyresample = "*" # Base requirement Geospatial image resampling # efficiency improvements >= 1.22.3 pyyaml = "*" # Base requirement @@ -78,12 +80,11 @@ pyaml-env = "*" # Reading YAML output config files, with paths pyorbital = "*" # required by satpy pypublicdecompwt = "*" # Required to read SEVIRI data rio-cogeo = "*" # Cloud Optimized GEOTIFF output -rasterio = "*" # GEOTIFF output -# version <1.3.10 required for bug in geotiff_standard -# Remove version requirement when bug fixed +rasterio = "*" # GEOTIFF output; Pinned to <1.3.10 due to the bug in geotiff_standard referencing = "*" satpy = "*" # efficiency improvements >= 0.33.1 scikit-image = "*" # Radius based center coverage checks +alphashape = ">=1.3.1" # For masking scipy's griddata output tabulate = "*" # Tables for the CLI colorama = "*" # Easy terminal color codes @@ -99,6 +100,12 @@ sphinxcontrib-autoprogram = { version = "*", optional = true } m2r2 = { version = "*", optional = true } brassy = { version = "*", optional = true } sphinx-argparse = { version = "*", optional = true } +# Version 3.0.2 causes error , 0.8.4 works +# Unsure why there was such a wide range of version numbers installed. +# File "lib/python3.10/site-packages/m2r2.py", line 82, in +# class RestBlockGrammar(mistune.BlockGrammar): +# AttributeError: module 'mistune' has no attribute 'BlockGrammar' +mistune = { version = "0.8.4", optional = true } # Lint group bandit = { version = "*", optional = true } black = { version = "*", optional = true } @@ -130,6 +137,7 @@ doc = [ "sphinxcontrib-autoprogram", # Required for adding command line options to documentation "m2r2", # Required for rendering markdown into RST-based documentation "brassy", # Release Note Generation + "mistune", # Required for rendering markdown into RST-based documentation ] lint = [ "bandit", # Syntax/security checking diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..f2e21dde9 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +addopts = -v -rf --ff --cov-report=term-missing +testpaths = + tests/unit_tests* +norecursedirs = xarray_utils \ No newline at end of file diff --git a/setup/check_system_requirements.sh b/setup/check_system_requirements.sh index fb81963af..fe48de77e 100755 --- a/setup/check_system_requirements.sh +++ b/setup/check_system_requirements.sh @@ -35,19 +35,6 @@ mkdir -p `dirname $install_log` echo "" echo "Install log: $install_log" -# These are the download locations used by the test_data function -test_data_urls=( - "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_fusion.tgz" - "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_noaa_aws.tgz" - "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_amsr2_1.6.0.tgz" - "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_clavrx_1.10.0.tgz" - "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_gpm_1.6.0.tgz" - "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_sar_1.12.2.tgz" - "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_scat_1.11.3.tgz" - "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_smap_1.6.0.tgz" - "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_viirs_1.6.0.tgz" -) - # Requirements to run base geoips tests if [[ "$1" == "geoips_base" ]]; then . $GEOIPS_PACKAGES_DIR/geoips/setup/check_system_requirements.sh git @@ -63,6 +50,10 @@ if [[ "$1" == "geoips_full" ]]; then . $GEOIPS_PACKAGES_DIR/geoips/setup/check_system_requirements.sh openblas fi +if [[ "$1" == "set_gitconfig" ]]; then + source $GEOIPS_PACKAGES_DIR/geoips/setup/bash_setup/gitconfigs +fi + if [[ "$1" == "run_command" ]]; then echo "Running $2 ... " `$2 >> $install_log 2>&1` @@ -328,12 +319,7 @@ if [[ "$1" == "test_data" || "$1" == "test_data_github" ]]; then test_data_dir=$GEOIPS_TESTDATA_DIR/$test_data_name test_data_url="$GEOIPS_REPO_URL/${test_data_name}.git" if [[ "$test_data_source_location" != "github" ]]; then - for url in ${test_data_urls[@]}; do - if [[ "${url}" == *"${test_data_name}"*".tgz" ]]; then - test_data_url="${url}" - break - fi - done + test_data_url="direct download" fi # Ensure there is a data or docs directory @@ -346,8 +332,11 @@ if [[ "$1" == "test_data" || "$1" == "test_data_github" ]]; then data_path="$test_data_dir/outputs/*" ls $data_path >> $install_log 2>&1 retval_outputs=$? + data_path="$test_data_dir/*.tgz" + ls $data_path >> $install_log 2>&1 + retval_tgz=$? - if [[ "$retval_data" != "0" && "$retval_docs" != "0" && "$retval_outputs" != "0" ]]; then + if [[ "$retval_data" != "0" && "$retval_docs" != "0" && "$retval_outputs" != "0" && "$retval_tgz" != "0" ]]; then if [[ "$exit_on_missing" == "true" ]]; then echo "FAILED: Missing $test_data_name_string" echo " Please run install script, then rerun test script. " @@ -356,9 +345,9 @@ if [[ "$1" == "test_data" || "$1" == "test_data_github" ]]; then exit 1 fi echo "Installing $test_data_name_string .... " - echo " $test_data_dir/ from $test_data_url via $test_data_source_location" + echo " $test_data_dir/ using $test_data_url via $test_data_source_location" if [[ "$test_data_source_location" == "github" ]]; then - python $SCRIPT_DIR/download_test_data.py $test_data_url $test_data_dir >> $install_log 2>&1 + python3 $SCRIPT_DIR/download_test_data.py $test_data_url --output-dir $GEOIPS_TESTDATA_DIR >> $install_log 2>&1 retval=$? if [[ "$retval" == "0" ]]; then echo "SUCCESS: Pulled ${test_data_name} from ${test_data_url}" @@ -378,10 +367,19 @@ if [[ "$1" == "test_data" || "$1" == "test_data_github" ]]; then echo "SUCCESS: successfully switch to branch $switch_to_branch" fi fi + if [[ -e $test_data_dir/uncompress_test_data.sh ]]; then + $test_data_dir/uncompress_test_data.sh >> $install_log 2>&1 + retval=$? + if [[ "$retval" == "0" ]]; then + echo "SUCCESS: Decompressed ${test_data_name}" + else + echo "FAILED: Failed to decompress ${test_data_name}. Try deleting and rerunning." + exit 1 + fi + fi else - echo "DOWNLOADING: NextCloud Dataset $test_data_name @ $test_data_url" - echo "python $SCRIPT_DIR/download_test_data.py $test_data_url $test_data_dir | tar -xz -C $GEOIPS_TESTDATA_DIR >> $install_log 2>&1" - python $SCRIPT_DIR/download_test_data.py $test_data_url $test_data_dir | tar -xz -C $GEOIPS_TESTDATA_DIR >> $install_log 2>&1 + echo "DOWNLOADING: NextCloud Dataset $test_data_name" + python3 $SCRIPT_DIR/download_test_data.py $test_data_name --output-dir $GEOIPS_TESTDATA_DIR # check to see how many folders in GEOIPS_TESTDATA_DIR match test_data_name matching_folders=$(ls $GEOIPS_TESTDATA_DIR | grep $test_data_name) folder_count=$(echo "$matching_folders" | wc -l) @@ -400,23 +398,12 @@ if [[ "$1" == "test_data" || "$1" == "test_data_github" ]]; then fi retval=$? if [[ "$retval" == "0" ]]; then - echo "SUCCESS: Decompressed ${test_data_name}" + echo "SUCCESS: Pulled ${test_data_name} from NexCloud ${test_data_url}" else - echo "FAILED: Failed to decompress ${test_data_name}" + echo "FAILED: Failed to pull ${test_data_name} from NexCloud ${test_data_url}" echo " try deleting and re-running" exit 1 fi - # If this is a github repo, then check if current-branch exists - # and switch to it if so. Allow branch not existing. - if [[ "$switch_to_branch" != "" ]]; then - echo "git -C $test_data_dir checkout $switch_to_branch >> $install_log 2>&1" - git -C $test_data_dir checkout $switch_to_branch >> $install_log 2>&1 - if [[ "$?" != "0" ]]; then - echo "Branch $switch_to_branch did not exist, staying on current branch" - else - echo "SUCCESS: successfully switch to branch $switch_to_branch" - fi - fi fi else echo "SUCCESS: $test_data_name_string appears to be installed successfully" diff --git a/setup/config_geoips b/setup/config_geoips index d05cf4a45..6b247dda9 100644 --- a/setup/config_geoips +++ b/setup/config_geoips @@ -39,7 +39,7 @@ export GEOIPS_PACKAGES_DIR=$GEOIPS_BASEDIR/geoips_packages export GEOIPS_TESTDATA_DIR=$GEOIPS_BASEDIR/test_data export CARTOPY_DATA_DIR=$GEOIPS_DEPENDENCIES_DIR/CARTOPY_DATA_DIR -export PYTHONPATH=$GEOIPS_BASEDIR/geoips_packages +# export PYTHONPATH=$GEOIPS_BASEDIR/geoips_packages export PATH=$GEOIPS_DEPENDENCIES_DIR/bin:$PATH ############################################################################ diff --git a/setup/download_test_data.py b/setup/download_test_data.py index 5fd99af96..1a4acab20 100644 --- a/setup/download_test_data.py +++ b/setup/download_test_data.py @@ -4,28 +4,297 @@ # # # https://github.com/NRLMMD-GEOIPS. """Download data from a specified URL.""" - -import sys +import subprocess import requests -from subprocess import check_output +import tarfile +import argparse +import os + +import yaml + + +def get_argparse_formatter(): + """ + Determine and return the appropriate argument parser help formatter. + + Tries to import ``RichHelpFormatter`` from the ``rich_argparse`` package. If the + import fails, it defaults to using the argparse default: ``argparse.HelpFormatter``. + + Returns + ------- + argparse.HelpFormatter or rich_argparse.RichHelpFormatter + The formatter class to use for argument parsing help messages. + """ + try: + from rich_argparse import RichHelpFormatter + + return RichHelpFormatter + except ModuleNotFoundError: + return argparse.HelpFormatter + + +def setup_rich_console(use_rich): + """ + Set up global ``output_to_console`` function using `rich` if available. + + Sets global variable ``output_to_console`` to a function that handles + styled output using ``rich`` if ``use_rich`` is True and ``rich`` is installed, + otherwise falls back to using standard print function. + + Parameters + ---------- + use_rich : bool + If True, attempts to use the `rich` library for styled console output. + + Returns + ------- + None + + Global Variables + ---------------- + output_to_console : function + A function to handle console output, with or without styling. + """ + global output_to_console + if use_rich: + try: + from rich.console import Console + + con = Console() + + def output_to_console(msg, style): + con.print(msg, style=style) + + return + except ModuleNotFoundError: + pass + + def output_to_console(msg, style): + print(msg) + + +def sizeof_fmt(num, suffix="B"): + """ + Convert a byte size into a human-readable string (eg. MiB). + + Parameters + ---------- + num : int + The size in bytes to convert. + suffix : str, optional + The suffix to append to the size (default is "B" for bytes). + + Returns + ------- + str + The size converted into a human-readable string with appropriate units. + + Notes + ----- + This function is adapted from: https://stackoverflow.com/a/1094933/2503170 + """ + for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"): + if abs(num) < 1024.0: + return f"{num:3.1f} {unit}{suffix}" + num /= 1024.0 + return f"{num:.1f}Yi{suffix}" + +def download_from_git(repo_url, destination): + """ + Clone a git repository from the specified URL to given destination. + + Parameters + ---------- + repo_url : str + The URL of the git repository to clone. + destination : str + The local directory where the repository should be cloned. + + Return + ------- + None + + Raises + ------ + subprocess.CalledProcessError + If the git clone command fails. + """ + try: + output_to_console( + f"Cloning repository from {repo_url} to {destination}", style="bold cyan" + ) + subprocess.check_output(["git", "clone", repo_url, destination]) + output_to_console("Repository successfully cloned.", style="bold green") + except subprocess.CalledProcessError as e: + output_to_console( + f"Failed to clone repository: {e.output.decode('utf-8')}", style="bold red" + ) + raise e -def download_test_data(url, dest=None): - """Download the specified URL and write to stdout as bytes. - Will raise requests.exceptions.HTTPError on failure. +def download_and_extract_compressed_tar(url, dest, comp="gz"): """ + Download a compressed tar file from a URL and extract its contents. + + This function streams the download of a compressed tar file, + supporting various compression types (e.g., gzip by default), + and extracts the files directly to the specified destination directory. + It does this in memory in chunks to prevent a memory overflow on large files and + for a speed increase by never having to re-read data off of slower non-RAM memory. + + Parameters + ---------- + url : str + The URL of the compressed tar file to download + dest : str + The directory where the contents of the tar file should be extracted. + If the directory does not exist, + it should be created prior to calling this function. + comp : str, optional + The compression type used in the tar file. Accepted values include: + - "gz" for gzip compression (default). + - "bz2" for bzip2 compression. + - "xz" for xz compression. + This parameter determines how the tar file will be read and decompressed. + + Returns + ------- + None + """ + output_to_console( + f"Downloading and extracting {url} to {dest}...", style="bold cyan" + ) + try: + with requests.get(url, stream=True, timeout=360) as r: + r.raise_for_status() + file_length = int(r.headers.get("content-length", 0)) + with tarfile.open(fileobj=r.raw, mode=f"r|{comp}") as tar: + output_to_console( + f"File is {sizeof_fmt(file_length)}... ", + style="cyan", + ) + # Trusting archives to not be malicious by not filtering files + tar.extractall(path=dest) # nosec + output_to_console("Success. Files downloaded and extracted.", style="green") + except Exception as e: + output_to_console("Failed to download or extract files.", style="bold red") + raise e + + +def get_test_data_urls(): + """ + Retrieve test data URLs from a YAML configuration file. + + This function reads a YAML file named `test-data-urls.yaml` located + in the same directory as the script and returns the URLs specified + under the `test_data_urls` key. + + Returns + ------- + list of str + A list of test data URLs. + + Examples + -------- + >>> urls = get_test_data_urls() + >>> print(urls) + ['https://example.com/data1.csv', 'https://example.com/data2.csv'] + """ + dirname, filename = os.path.split(os.path.abspath(__file__)) + with open(os.path.join(dirname, "test-data-urls.yaml"), "r") as f: + data = yaml.safe_load(f) + return data["test_data_urls"] + + +def main(): + """ + Handle command-line arguments and initiate the download process. + + Determines what function to call based off of input value. + Also configures the console output style based on user preferences. + + Returns + ------- + None + """ + parser = argparse.ArgumentParser( + description="Download test data for GeoIPS", + formatter_class=get_argparse_formatter(), + ) + parser.add_argument( + "input", + help="The test data set to download, URL to the .tgz file or URL to git repo.", + nargs="?", + default=None, + ) + parser.add_argument( + "--output-dir", + help="The directory to extract/clone files to.", + default=None, + ) + parser.add_argument( + "--no-rich", + action="store_true", + help="Disable rich text formatting and progress bars.", + ) + parser.add_argument( + "--test-data-available", + action="store_true", + help="Returns 0 if available, else 1.", + ) + parser.add_argument( + "--list-test-datasets", + action="store_true", + default=False, + help="List test data sets available for download.", + required=False, + ) + + args = parser.parse_args() + + use_rich = not args.no_rich + setup_rich_console(use_rich) + + test_data_urls = get_test_data_urls() + if args.test_data_available: + if args.input in test_data_urls.keys(): + output_to_console( + f"{args.input} is available for direct download.", style="green" + ) + exit() + else: + output_to_console( + f"{args.input} is NOT available for direct download.", style="red" + ) + exit(1) + if args.list_test_datasets: + output_to_console("Available data sets:\n", style="") + for key in test_data_urls.keys(): + output_to_console(f"\t{key}", style="cyan") + exit() + + if not (args.input and args.output_dir): + output_to_console("Invalid arguments.", style="bold red") + parser.print_help() + exit(1) + + if args.input in test_data_urls.keys(): + output_to_console(f"Recognized test data set [{args.input}]", style="cyan") + url = test_data_urls[args.input] + output_to_console(f"Trying to download from url {url}", style="cyan") + else: + url = args.input + if ".git" in url: - print(f"git clone {url} {dest}") - check_output(["git", "clone", url, dest]) - print("done git clone") + download_from_git(url, args.output_dir) + elif ".tgz" in url: + download_and_extract_compressed_tar(url, args.output_dir) else: - resp = requests.get(url, stream=True, timeout=15) - sys.stdout.buffer.write(resp.raw.read()) + output_to_console( + "Error: Cannot handle non-git non-tgz urls.", style="bold red" + ) if __name__ == "__main__": - dest = None - if len(sys.argv) == 3: - dest = sys.argv[2] - download_test_data(sys.argv[1], dest) + main() diff --git a/setup/geoips_conda_init_setup b/setup/geoips_conda_init_setup index f34392fcf..490ef1ab8 100644 --- a/setup/geoips_conda_init_setup +++ b/setup/geoips_conda_init_setup @@ -10,6 +10,7 @@ else GEOIPS_CONDA_DIR=$GEOIPS_DEPENDENCIES_DIR/miniconda3 fi +# DO NOT set PATH in here. Ever. if [[ -f $GEOIPS_CONDA_DIR/bin/conda ]]; then # >>> conda initialize >>> # !! Contents within this block are managed by 'conda init' !! @@ -19,21 +20,17 @@ if [[ -f $GEOIPS_CONDA_DIR/bin/conda ]]; then else if [ -f "$GEOIPS_CONDA_DIR/etc/profile.d/conda.sh" ]; then . "$GEOIPS_CONDA_DIR/etc/profile.d/conda.sh" - else - export PATH="$GEOIPS_DEPENDENCIES_DIR/miniconda3/bin:$PATH" + elif [ -f "$GEOIPS_DEPENDENCIES_DIR/miniconda3/etc/profile.d/mamba.sh" ]; then + . "$GEOIPS_DEPENDENCIES_DIR/miniconda3/etc/profile.d/mamba.sh" fi fi unset __conda_setup # <<< conda initialize <<< fi -export PATH="$GEOIPS_DEPENDENCIES_DIR/miniconda3/bin:$PATH" whichconda=`which conda 2> /dev/null` if [[ "$whichconda" != "" ]]; then if conda env list | grep -q --regex '^geoips_conda'; then eval "$(conda shell.bash activate geoips_conda)" fi fi -if [ -f "$GEOIPS_DEPENDENCIES_DIR/miniconda3/etc/profile.d/mamba.sh" ]; then - . "$GEOIPS_DEPENDENCIES_DIR/miniconda3/etc/profile.d/mamba.sh" -fi diff --git a/setup/test-data-urls.yaml b/setup/test-data-urls.yaml new file mode 100644 index 000000000..aa7b1b14d --- /dev/null +++ b/setup/test-data-urls.yaml @@ -0,0 +1,10 @@ +test_data_urls: + test_data_fusion: "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_fusion.tgz" + test_data_noaa_aws: "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_noaa_aws.tgz" + test_data_amsr2: "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_amsr2_1.6.0.tgz" + test_data_clavrx: "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_clavrx_1.10.0.tgz" + test_data_gpm: "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_gpm_1.6.0.tgz" + test_data_sar: "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_sar_1.12.2.tgz" + test_data_scat: "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_scat_1.11.3.tgz" + test_data_smap: "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_smap_1.6.0.tgz" + test_data_viirs: "https://io2.cira.colostate.edu/s/J73tEcn22smktMi/download?path=%2F&files=test_data_viirs_1.6.0.tgz" diff --git a/tests/download_noaa_aws.sh b/tests/download_noaa_aws.sh index 5410d03f3..c90b51367 100755 --- a/tests/download_noaa_aws.sh +++ b/tests/download_noaa_aws.sh @@ -25,13 +25,39 @@ if [[ "$8" == "" && -z "$GEOIPS_PACKAGES_DIR" ]]; then fi if [[ "$1" == "" || "$bad_command" == "1" ]]; then - echo "Usage: $0 YYYY MM DD HH MN " - echo " satellite: goes16, goes17, himawari8, himawari9, geokompsat" - echo " testdata_dir: if 'default' or not specified, defaults to: " + echo "Usage: $0 YYYY MM DD HH MN " + echo " satellite:" + echo " goes16" + echo " goes17" + echo " himawari8" + echo " himawari9" + echo " geokompsat" + echo " noaa-20" + echo " noaa-21" + echo " snpp" + echo " jpss" + echo " testdata_dir: " + echo " if 'default' or not specified, defaults to: " echo " \$GEOIPS_TESTDATA_DIR/test_data_noaa_aws/data///" - echo " rclone_conf: if 'default' or not specified, defaults to:" + echo " rclone_conf: " + echo " if 'default' or not specified, defaults to:" echo " \$GEOIPS_PACKAGES_DIR/geoips/setup/rclone_setup/rclone.conf" - echo " wildcard_list: list of strings to match in filenames (ie, channels). Defaults to all files for dtg" + echo " collection:" + echo " If not defined, defaults to L1B full disk for geostationary" + echo " Since there are so many NOAA/NPP products, no sensible default.." + echo " This refers to the initial subdirectory found in the NOAA AWS" + echo " S3 buckets (prior to the date-based subdirs)" + echo " ie: noaa-nesdis-snpp-pds.s3.amazonaws.com/VIIRS-IMG-GEO-TC" + echo " examples of collections (you can find all available collections" + echo " by navigating the S3 buckets on the web)" + echo " viirs" + echo " VIIRS-IMG-GEO-TC" + echo " VIIRS-I5-SDR" + echo " ahi" + echo " AHI-L1b-FLDK" + echo " wildcard_list: " + echo " list of strings to match in filenames (ie, channels)." + echo " Defaults to all files for dtg" exit 1 fi @@ -87,7 +113,6 @@ elif [[ "$satellite" == "geokompsat" ]]; then echo "rclone --config $rclone_conf lsf $rclone_path" files=`rclone --config $rclone_conf lsf $rclone_path` echo "COMPARE: ${yyyy}${mm}${dd}${hh}${mn}" - break for fname in $files; do if [[ "$fname" =~ "${yyyy}${mm}${dd}${hh}${mn}" ]]; then if [[ "$wildcard_list" == "" ]]; then @@ -103,6 +128,42 @@ elif [[ "$satellite" == "geokompsat" ]]; then fi fi done +elif [[ "$satellite" == "noaa-20" || "$satellite" == "noaa-21" || "$satellite" == "jpss" || "$satellite" == "snpp" ]]; then + collection=${collection:-"VIIRS-I1-SDR"} + # https://noaa-nesdis-n20-pds.s3.amazonaws.com/VIIRS-I1-SDR/2024/08/07/SVI01_j01_d20240807_t0000596_e0002241_b34811_c20240807002901054000_oebc_ops.h5 + if [[ "$satellite" == "noaa-20" ]]; then + resource_name="noaa-nesdis-n20-pds" + elif [[ "$satellite" == "noaa-21" ]]; then + resource_name="noaa-nesdis-n21-pds" + elif [[ "$satellite" == "snpp" ]]; then + resource_name="noaa-nesdis-snpp-pds" + elif [[ "$satellite" == "jpss" ]]; then + resource_name="noaa-nesdis-jpss" + fi + rclone_path="publicAWS:$resource_name/$collection/$yyyy/$mm/$dd/" + echo "" + echo "************************************************************************************************************" + echo "URL listing available files: https://${resource_name}.s3.amazonaws.com/index.html#$collection/$yyyy/$mm/$dd/" + echo "************************************************************************************************************" + echo "" + echo "rclone --config $rclone_conf lsf $rclone_path" + files=`rclone --config $rclone_conf lsf $rclone_path` + echo "COMPARE: d${yyyy}${mm}${dd}_t${hh}${mn}" + for fname in $files; do + if [[ "$fname" =~ "d${yyyy}${mm}${dd}_t${hh}${mn}" ]]; then + if [[ "$wildcard_list" == "" ]]; then + echo "rclone --config $rclone_conf copy -P $rclone_path/$fname $testdata_dir/" + rclone --config $rclone_conf copy -P $rclone_path/$fname $testdata_dir/ + else + for wildcard in $wildcard_list; do + if [[ "$fname" =~ "$wildcard" ]]; then + echo "rclone --config $rclone_conf copy -P $rclone_path/$fname $testdata_dir/" + rclone --config $rclone_conf copy -P $rclone_path/$fname $testdata_dir/ + fi + done + fi + fi + done else collection=${collection:-"ABI-L1b-RadF"} rclone_path="publicAWS:noaa-$satellite/$collection/$yyyy/$jday/$hh/" diff --git a/tests/integration_tests/base_install.sh b/tests/integration_tests/base_install.sh index 7758881bf..bd0d4a96d 100755 --- a/tests/integration_tests/base_install.sh +++ b/tests/integration_tests/base_install.sh @@ -12,4 +12,3 @@ fi . $GEOIPS_PACKAGES_DIR/geoips/setup/check_system_requirements.sh geoips_base . $GEOIPS_PACKAGES_DIR/geoips/setup/check_system_requirements.sh test_data test_data_amsr2 $test_exit $install_script -. $GEOIPS_PACKAGES_DIR/geoips/setup/check_system_requirements.sh test_data test_data_noaa_aws $test_exit $install_script diff --git a/tests/outputs/abi.static.Infrared.imagery_annotated/20200918.195020.goes-16.abi.Infrared.goes_east.45p56.noaa.10p0.png b/tests/outputs/abi.static.Infrared.imagery_annotated/20200918.195020.goes-16.abi.Infrared.goes_east.45p56.noaa.10p0.png index 05f8da86f..8146a868d 100644 Binary files a/tests/outputs/abi.static.Infrared.imagery_annotated/20200918.195020.goes-16.abi.Infrared.goes_east.45p56.noaa.10p0.png and b/tests/outputs/abi.static.Infrared.imagery_annotated/20200918.195020.goes-16.abi.Infrared.goes_east.45p56.noaa.10p0.png differ diff --git a/tests/outputs/abi.static.Visible.imagery_annotated/20200918.195020.goes-16.abi.Visible.goes_east.41p12.noaa.10p0.png b/tests/outputs/abi.static.Visible.imagery_annotated/20200918.195020.goes-16.abi.Visible.goes_east.41p12.noaa.10p0.png index 0bbc7eb80..6fe68a766 100644 Binary files a/tests/outputs/abi.static.Visible.imagery_annotated/20200918.195020.goes-16.abi.Visible.goes_east.41p12.noaa.10p0.png and b/tests/outputs/abi.static.Visible.imagery_annotated/20200918.195020.goes-16.abi.Visible.goes_east.41p12.noaa.10p0.png differ diff --git a/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png b/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png index 5ac1ffad5..ce6686ae1 100644 Binary files a/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png and b/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png differ diff --git a/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png b/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png index f10181ffb..1e844d3f7 100644 Binary files a/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png and b/tests/outputs/amsr2.global_overlay.37pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.37pct.global.10p06.star.20p0.png differ diff --git a/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png b/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png index 125a02e20..5a1ac7197 100644 Binary files a/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png and b/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png differ diff --git a/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png b/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png index 69b993814..0badc6ed9 100644 Binary files a/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png and b/tests/outputs/amsr2.global_overlay.89pct.imagery_annotated_over_Visible/20200518.062048.gcom-w1.amsr2.89pct.global.13p55.star.20p0.png differ diff --git a/tests/outputs/amsr2.tc.89H-Physical.imagery_annotated/20200518_073601_IO012020_amsr2_gcom-w1_89H-Physical_140kts_100p00_res1p0-cr300.png b/tests/outputs/amsr2.tc.89H-Physical.imagery_annotated/20200518_073601_IO012020_amsr2_gcom-w1_89H-Physical_140kts_100p00_res1p0-cr300.png index 25be9d23c..1957d803f 100644 Binary files a/tests/outputs/amsr2.tc.89H-Physical.imagery_annotated/20200518_073601_IO012020_amsr2_gcom-w1_89H-Physical_140kts_100p00_res1p0-cr300.png and b/tests/outputs/amsr2.tc.89H-Physical.imagery_annotated/20200518_073601_IO012020_amsr2_gcom-w1_89H-Physical_140kts_100p00_res1p0-cr300.png differ diff --git a/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgInfrared-Gray.png b/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgInfrared-Gray.png index a6bdabc33..e7004de57 100644 Binary files a/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgInfrared-Gray.png and b/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgInfrared-Gray.png differ diff --git a/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgVisible.png b/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgVisible.png index 273fac665..b357b1342 100644 Binary files a/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgVisible.png and b/tests/outputs/amsr2.tc_overlay.37pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_37pct_140kts_95p89_res1p0-cr100-bgVisible.png differ diff --git a/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgInfrared-Gray.png b/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgInfrared-Gray.png index c223173c0..f41d53cf1 100644 Binary files a/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgInfrared-Gray.png and b/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Infrared-Gray/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgInfrared-Gray.png differ diff --git a/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgVisible.png b/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgVisible.png index 31bbb8054..0e1c03478 100644 Binary files a/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgVisible.png and b/tests/outputs/amsr2.tc_overlay.89pct.imagery_annotated_over_Visible/20200518_073601_IO012020_amsr2_gcom-w1_89pct_140kts_98p32_res1p0-cr100-bgVisible.png differ diff --git a/tests/outputs/amsr2_ocean.tc.windspeed.imagery_clean/20200518_073601_IO012020_amsr2_gcom-w1_windspeed_140kts_85p45_1p0-clean.png b/tests/outputs/amsr2_ocean.tc.windspeed.imagery_clean/20200518_073601_IO012020_amsr2_gcom-w1_windspeed_140kts_85p45_1p0-clean.png index ac69b0765..189e4a16d 100644 Binary files a/tests/outputs/amsr2_ocean.tc.windspeed.imagery_clean/20200518_073601_IO012020_amsr2_gcom-w1_windspeed_140kts_85p45_1p0-clean.png and b/tests/outputs/amsr2_ocean.tc.windspeed.imagery_clean/20200518_073601_IO012020_amsr2_gcom-w1_windspeed_140kts_85p45_1p0-clean.png differ diff --git a/tests/outputs/ascat_knmi.tc.windbarbs.imagery_windbarbs_clean/20210421_014248_WP022021_ascat_metop-c_windbarbs_120kts_78p20_0p5-clean.png b/tests/outputs/ascat_knmi.tc.windbarbs.imagery_windbarbs_clean/20210421_014248_WP022021_ascat_metop-c_windbarbs_120kts_78p20_0p5-clean.png index 93a5fd3cd..83a0876f2 100644 Binary files a/tests/outputs/ascat_knmi.tc.windbarbs.imagery_windbarbs_clean/20210421_014248_WP022021_ascat_metop-c_windbarbs_120kts_78p20_0p5-clean.png and b/tests/outputs/ascat_knmi.tc.windbarbs.imagery_windbarbs_clean/20210421_014248_WP022021_ascat_metop-c_windbarbs_120kts_78p20_0p5-clean.png differ diff --git a/tests/outputs/ascat_low_knmi.tc.windbarbs.imagery_windbarbs/20210421_014156_WP022021_ascat_metop-c_windbarbs_120kts_35p17_1p0.png b/tests/outputs/ascat_low_knmi.tc.windbarbs.imagery_windbarbs/20210421_014156_WP022021_ascat_metop-c_windbarbs_120kts_35p17_1p0.png index e8f09633d..77ce8d059 100644 Binary files a/tests/outputs/ascat_low_knmi.tc.windbarbs.imagery_windbarbs/20210421_014156_WP022021_ascat_metop-c_windbarbs_120kts_35p17_1p0.png and b/tests/outputs/ascat_low_knmi.tc.windbarbs.imagery_windbarbs/20210421_014156_WP022021_ascat_metop-c_windbarbs_120kts_35p17_1p0.png differ diff --git a/tests/outputs/ascat_noaa_25km.tc.windbarbs.imagery_windbarbs/20230524_235304_WP022023_ascat_metop-c_windbarbs_135kts_39p90_0p7.png b/tests/outputs/ascat_noaa_25km.tc.windbarbs.imagery_windbarbs/20230524_235304_WP022023_ascat_metop-c_windbarbs_135kts_39p90_0p7.png index 3ad2822b4..9977854a9 100644 Binary files a/tests/outputs/ascat_noaa_25km.tc.windbarbs.imagery_windbarbs/20230524_235304_WP022023_ascat_metop-c_windbarbs_135kts_39p90_0p7.png and b/tests/outputs/ascat_noaa_25km.tc.windbarbs.imagery_windbarbs/20230524_235304_WP022023_ascat_metop-c_windbarbs_135kts_39p90_0p7.png differ diff --git a/tests/outputs/ascat_noaa_50km.tc.wind-ambiguities.imagery_windbarbs/20230524_235200_WP022023_ascat_metop-c_wind-ambiguities_135kts_50p08_1p1.png b/tests/outputs/ascat_noaa_50km.tc.wind-ambiguities.imagery_windbarbs/20230524_235200_WP022023_ascat_metop-c_wind-ambiguities_135kts_50p08_1p1.png index 06b592556..4e897b214 100644 Binary files a/tests/outputs/ascat_noaa_50km.tc.wind-ambiguities.imagery_windbarbs/20230524_235200_WP022023_ascat_metop-c_wind-ambiguities_135kts_50p08_1p1.png and b/tests/outputs/ascat_noaa_50km.tc.wind-ambiguities.imagery_windbarbs/20230524_235200_WP022023_ascat_metop-c_wind-ambiguities_135kts_50p08_1p1.png differ diff --git a/tests/outputs/ascat_uhr.tc.wind-ambiguities.imagery_windbarbs/20210421_014200_WP022021_ascatuhr_metop-c_wind-ambiguities_120kts_100p00_0p1.png b/tests/outputs/ascat_uhr.tc.wind-ambiguities.imagery_windbarbs/20210421_014200_WP022021_ascatuhr_metop-c_wind-ambiguities_120kts_100p00_0p1.png index f9868dc95..4ab800173 100644 Binary files a/tests/outputs/ascat_uhr.tc.wind-ambiguities.imagery_windbarbs/20210421_014200_WP022021_ascatuhr_metop-c_wind-ambiguities_120kts_100p00_0p1.png and b/tests/outputs/ascat_uhr.tc.wind-ambiguities.imagery_windbarbs/20210421_014200_WP022021_ascatuhr_metop-c_wind-ambiguities_120kts_100p00_0p1.png differ diff --git a/tests/outputs/ascat_uhr.tc.windbarbs.imagery_windbarbs/20230722_234513_AL052023_ascatuhr_metop-b_windbarbs_65kts_63p79_1p0.png b/tests/outputs/ascat_uhr.tc.windbarbs.imagery_windbarbs/20230722_234513_AL052023_ascatuhr_metop-b_windbarbs_65kts_63p79_1p0.png index 85ba01a6d..7e6715ff1 100644 Binary files a/tests/outputs/ascat_uhr.tc.windbarbs.imagery_windbarbs/20230722_234513_AL052023_ascatuhr_metop-b_windbarbs_65kts_63p79_1p0.png and b/tests/outputs/ascat_uhr.tc.windbarbs.imagery_windbarbs/20230722_234513_AL052023_ascatuhr_metop-b_windbarbs_65kts_63p79_1p0.png differ diff --git a/tests/outputs/ascat_uhr.tc.windspeed.imagery_clean/20230912_011802_AL132023_ascatuhr_metop-c_windspeed_100kts_51p44_1p0.png b/tests/outputs/ascat_uhr.tc.windspeed.imagery_clean/20230912_011802_AL132023_ascatuhr_metop-c_windspeed_100kts_51p44_1p0.png index 82ef9a68e..c1ab41c6a 100644 Binary files a/tests/outputs/ascat_uhr.tc.windspeed.imagery_clean/20230912_011802_AL132023_ascatuhr_metop-c_windspeed_100kts_51p44_1p0.png and b/tests/outputs/ascat_uhr.tc.windspeed.imagery_clean/20230912_011802_AL132023_ascatuhr_metop-c_windspeed_100kts_51p44_1p0.png differ diff --git a/tests/outputs/cygnss.tc.windspeed.imagery_clean/20240125_060307_SH062024_cygnss_cygnss_windspeed_80kts_6p22_1p0-clean.png b/tests/outputs/cygnss.tc.windspeed.imagery_clean/20240125_060307_SH062024_cygnss_cygnss_windspeed_80kts_6p22_1p0-clean.png new file mode 100644 index 000000000..371c272b9 Binary files /dev/null and b/tests/outputs/cygnss.tc.windspeed.imagery_clean/20240125_060307_SH062024_cygnss_cygnss_windspeed_80kts_6p22_1p0-clean.png differ diff --git a/tests/outputs/hy2.tc.windspeed.imagery_annotated/20211202_084039_WP272021_hscat_hy-2b_windspeed_95kts_97p06_1p0.png b/tests/outputs/hy2.tc.windspeed.imagery_annotated/20211202_084039_WP272021_hscat_hy-2b_windspeed_95kts_97p06_1p0.png index 37d84155d..5fdfd3dfb 100644 Binary files a/tests/outputs/hy2.tc.windspeed.imagery_annotated/20211202_084039_WP272021_hscat_hy-2b_windspeed_95kts_97p06_1p0.png and b/tests/outputs/hy2.tc.windspeed.imagery_annotated/20211202_084039_WP272021_hscat_hy-2b_windspeed_95kts_97p06_1p0.png differ diff --git a/tests/outputs/oscat_knmi.tc.windbarbs.imagery_windbarbs/20210209_025351_SH192021_oscat_scatsat-1_windbarbs_135kts_75p10_1p0.png b/tests/outputs/oscat_knmi.tc.windbarbs.imagery_windbarbs/20210209_025351_SH192021_oscat_scatsat-1_windbarbs_135kts_75p10_1p0.png index a4dfad843..2d5d736cc 100644 Binary files a/tests/outputs/oscat_knmi.tc.windbarbs.imagery_windbarbs/20210209_025351_SH192021_oscat_scatsat-1_windbarbs_135kts_75p10_1p0.png and b/tests/outputs/oscat_knmi.tc.windbarbs.imagery_windbarbs/20210209_025351_SH192021_oscat_scatsat-1_windbarbs_135kts_75p10_1p0.png differ diff --git a/tests/outputs/smap.tc.windspeed.imagery_clean/20210926_210400_WP202021_smap-spd_smap_windspeed_100kts_74p87_1p0-clean.png b/tests/outputs/smap.tc.windspeed.imagery_clean/20210926_210400_WP202021_smap-spd_smap_windspeed_100kts_74p87_1p0-clean.png index 8d1b8e4da..5322afa7e 100644 Binary files a/tests/outputs/smap.tc.windspeed.imagery_clean/20210926_210400_WP202021_smap-spd_smap_windspeed_100kts_74p87_1p0-clean.png and b/tests/outputs/smap.tc.windspeed.imagery_clean/20210926_210400_WP202021_smap-spd_smap_windspeed_100kts_74p87_1p0-clean.png differ diff --git a/tests/scripts/atms.tc.165H.netcdf_geoips.sh b/tests/scripts/atms.tc.165H.netcdf_geoips.sh index 4d8048d72..5543ebf35 100755 --- a/tests/scripts/atms.tc.165H.netcdf_geoips.sh +++ b/tests/scripts/atms.tc.165H.netcdf_geoips.sh @@ -31,7 +31,7 @@ geoips run single_source \ --compare_path "$GEOIPS_PACKAGES_DIR/geoips/tests/outputs/atms.tc..netcdf_geoips" \ --tc_spec_template tc_4km_256x256 \ --product_spec_override '{}' \ - --output_formatter_kwargs '{}' \ + --output_formatter_kwargs '{"clobber": "True"}' \ --filename_formatter_kwargs '{}' \ --metadata_output_formatter_kwargs '{}' \ --metadata_filename_formatter_kwargs '{}' diff --git a/tests/scripts/cygnss.tc.windspeed.imagery_clean.sh b/tests/scripts/cygnss.tc.windspeed.imagery_clean.sh new file mode 100755 index 000000000..8e4b3b030 --- /dev/null +++ b/tests/scripts/cygnss.tc.windspeed.imagery_clean.sh @@ -0,0 +1,19 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + +geoips run single_source \ + $GEOIPS_TESTDATA_DIR/test_data_cygnss/data/cyg.ddmi.s20240125-000001-e20240125-235959.l2.wind_trackgridsize25km_NOAAv1.2_L1a21.d21.nc \ + --reader_name cygnss_netcdf \ + --product_name windspeed \ + --output_formatter imagery_clean \ + --filename_formatter tc_clean_fname \ + --window_start_time "20240125T0600Z" \ + --window_end_time "20240125T0615Z" \ + --minimum_coverage 0 \ + --trackfile_parser bdeck_parser \ + --trackfiles $GEOIPS_PACKAGES_DIR/geoips/tests/sectors/tc_bdecks/bsh062024.dat \ + --compare_path "$GEOIPS_PACKAGES_DIR/geoips/tests/outputs/cygnss.tc.windspeed.imagery_clean" \ + +ss_retval=$? + +exit $((ss_retval)) \ No newline at end of file diff --git a/tests/sectors/tc_bdecks/bio012024.dat b/tests/sectors/tc_bdecks/bio012024.dat new file mode 100644 index 000000000..e81f6ccf7 --- /dev/null +++ b/tests/sectors/tc_bdecks/bio012024.dat @@ -0,0 +1,25 @@ +IO, 01, 2024052312, , BEST, 0, 136N, 866E, 20, 1001, DB, 0, , 0, 0, 0, 0, 1003, 150, 0, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052318, , BEST, 0, 143N, 876E, 20, 999, DB, 0, , 0, 0, 0, 0, 1002, 280, 0, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052400, , BEST, 0, 149N, 884E, 20, 997, DB, 0, , 0, 0, 0, 0, 999, 185, 110, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052406, , BEST, 0, 157N, 891E, 25, 996, TD, 0, , 0, 0, 0, 0, 1000, 205, 100, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052412, , BEST, 0, 164N, 894E, 25, 996, TD, 0, , 0, 0, 0, 0, 1000, 205, 90, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052418, , BEST, 0, 170N, 895E, 25, 996, TD, 0, , 0, 0, 0, 0, 1002, 220, 90, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052500, , BEST, 0, 177N, 894E, 25, 993, TD, 0, , 0, 0, 0, 0, 1000, 220, 80, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052506, , BEST, 0, 180N, 893E, 30, 993, TD, 0, , 0, 0, 0, 0, 998, 220, 80, 0, 0, B, 0, , 0, 0, INVEST, S, +IO, 01, 2024052512, , BEST, 0, 183N, 892E, 35, 990, TS, 34, NEQ, 105, 130, 150, 0, 998, 220, 70, 0, 0, B, 0, , 0, 0, ONE, S, 0, , 0, 0, 0, 0, TRANSITIONED, ioA92024 to io012024, +IO, 01, 2024052518, , BEST, 0, 189N, 891E, 35, 990, TS, 34, NEQ, 115, 125, 0, 0, 1000, 220, 75, 0, 0, B, 0, , 0, 0, REMAL, M, +IO, 01, 2024052600, , BEST, 0, 195N, 890E, 45, 990, TS, 34, NEQ, 115, 125, 0, 0, 1000, 220, 75, 0, 0, B, 0, , 0, 0, REMAL, M, +IO, 01, 2024052606, , BEST, 0, 202N, 888E, 55, 985, TS, 34, NEQ, 165, 130, 130, 95, 998, 190, 65, 0, 0, B, 0, , 0, 0, REMAL, M, +IO, 01, 2024052606, , BEST, 0, 202N, 888E, 55, 985, TS, 50, NEQ, 80, 90, 90, 0, 998, 190, 65, 0, 0, B, 0, , 0, 0, REMAL, M, +IO, 01, 2024052612, , BEST, 0, 209N, 887E, 55, 980, TS, 34, NEQ, 145, 130, 130, 105, 998, 180, 55, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052612, , BEST, 0, 209N, 887E, 55, 980, TS, 50, NEQ, 70, 75, 80, 30, 998, 180, 55, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052618, , BEST, 0, 218N, 886E, 55, 978, TS, 34, NEQ, 145, 130, 130, 105, 996, 160, 50, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052618, , BEST, 0, 218N, 886E, 55, 978, TS, 50, NEQ, 70, 75, 80, 30, 996, 160, 50, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052700, , BEST, 0, 226N, 889E, 50, 979, TS, 34, NEQ, 145, 130, 130, 105, 995, 200, 50, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052700, , BEST, 0, 226N, 889E, 50, 979, TS, 50, NEQ, 70, 75, 80, 30, 995, 200, 50, 0, 0, B, 0, , 0, 0, REMAL, D, +IO, 01, 2024052706, , BEST, 0, 232N, 893E, 45, 981, TS, 34, NEQ, 120, 135, 120, 85, 995, 170, 60, 0, 0, B, 0, , 0, 0, REMAL, , +IO, 01, 2024052712, , BEST, 0, 237N, 899E, 40, 983, TS, 34, NEQ, 105, 165, 135, 85, 995, 155, 65, 0, 0, B, 0, , 0, 0, REMAL, , +IO, 01, 2024052718, , BEST, 0, 242N, 904E, 40, 984, TS, 34, NEQ, 110, 170, 140, 90, 996, 130, 65, 0, 0, B, 0, , 0, 0, REMAL, M, +IO, 01, 2024052800, , BEST, 0, 249N, 912E, 40, 985, TS, 34, NEQ, 115, 175, 145, 95, 995, 105, 70, 0, 0, B, 0, , 0, 0, REMAL, , +IO, 01, 2024052806, , BEST, 0, 254N, 919E, 30, 990, TD, 0, , 0, 0, 0, 0, 998, 95, 60, 0, 0, B, 0, , 0, 0, REMAL, , +IO, 01, 2024052812, , BEST, 0, 263N, 931E, 30, 990, TD, 0, , 0, 0, 0, 0, 998, 95, 60, 0, 0, B, 0, , 0, 0, REMAL, , diff --git a/tests/sectors/tc_bdecks/bsh062024.dat b/tests/sectors/tc_bdecks/bsh062024.dat new file mode 100644 index 000000000..455c713c5 --- /dev/null +++ b/tests/sectors/tc_bdecks/bsh062024.dat @@ -0,0 +1,168 @@ +SH, 06, 2024011012, , BEST, 0, 59S, 868E, 15, 1008, DB, 0, , 0, 0, 0, 0, 1009, 155, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011018, , BEST, 0, 62S, 880E, 15, 1008, DB, 0, , 0, 0, 0, 0, 1009, 155, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011100, , BEST, 0, 65S, 891E, 15, 1008, DB, 0, , 0, 0, 0, 0, 1009, 155, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011106, , BEST, 0, 71S, 899E, 15, 1008, DB, 0, , 0, 0, 0, 0, 1009, 155, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011112, , BEST, 0, 76S, 905E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1009, 255, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011118, , BEST, 0, 81S, 911E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1009, 255, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011200, , BEST, 0, 87S, 917E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1009, 255, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011206, , BEST, 0, 95S, 925E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1009, 280, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011212, , BEST, 0, 100S, 929E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1008, 200, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011218, , BEST, 0, 105S, 932E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1008, 200, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011300, , BEST, 0, 108S, 934E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1008, 200, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011306, , BEST, 0, 112S, 936E, 20, 1007, DB, 0, , 0, 0, 0, 0, 1008, 200, 0, 0, 0, S, 0, , 0, 0, INVEST, , +SH, 06, 2024011312, , BEST, 0, 115S, 938E, 25, 1006, TD, 0, , 0, 0, 0, 0, 1008, 210, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011318, , BEST, 0, 116S, 940E, 25, 1005, TD, 0, , 0, 0, 0, 0, 1009, 210, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011400, , BEST, 0, 112S, 940E, 25, 1006, TD, 0, , 0, 0, 0, 0, 1008, 210, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011406, , BEST, 0, 108S, 938E, 25, 1005, TD, 0, , 0, 0, 0, 0, 1008, 180, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011412, , BEST, 0, 106S, 938E, 25, 1005, TD, 0, , 0, 0, 0, 0, 1008, 180, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011418, , BEST, 0, 104S, 937E, 30, 1005, TD, 0, , 0, 0, 0, 0, 1008, 180, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011500, , BEST, 0, 100S, 936E, 30, 1004, TD, 0, , 0, 0, 0, 0, 1008, 180, 55, 0, 0, S, 0, , 0, 0, INVEST, S, +SH, 06, 2024011506, , BEST, 0, 96S, 936E, 35, 1001, TS, 34, NEQ, 20, 50, 50, 75, 1008, 170, 35, 0, 0, S, 0, , 0, 0, INVEST, M, 0, , 0, 0, 0, 0, TRANSITIONED, shA82024 to sh062024, +SH, 06, 2024011512, , BEST, 0, 95S, 937E, 40, 1001, TS, 34, NEQ, 65, 55, 50, 55, 1008, 170, 35, 0, 0, S, 0, , 0, 0, SIX, M, +SH, 06, 2024011518, , BEST, 0, 96S, 938E, 40, 998, TS, 34, NEQ, 65, 45, 50, 65, 1008, 155, 25, 0, 0, S, 0, , 0, 0, SIX, M, +SH, 06, 2024011600, , BEST, 0, 95S, 939E, 40, 998, TS, 34, NEQ, 65, 45, 50, 65, 1008, 180, 25, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011606, , BEST, 0, 95S, 938E, 45, 995, TS, 34, NEQ, 65, 45, 50, 65, 1008, 200, 15, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011612, , BEST, 0, 96S, 940E, 45, 994, TS, 34, NEQ, 65, 45, 50, 65, 1007, 230, 15, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011618, , BEST, 0, 96S, 942E, 45, 994, TS, 34, NEQ, 65, 45, 50, 65, 1007, 230, 15, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011700, , BEST, 0, 98S, 941E, 50, 997, TS, 34, NEQ, 65, 45, 50, 65, 1007, 250, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011706, , BEST, 0, 101S, 940E, 50, 997, TS, 34, NEQ, 65, 45, 50, 65, 1007, 250, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011706, , BEST, 0, 101S, 940E, 50, 997, TS, 50, NEQ, 40, 0, 0, 0, 1007, 250, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011712, , BEST, 0, 104S, 940E, 45, 997, TS, 34, NEQ, 65, 45, 50, 65, 1008, 320, 50, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011718, , BEST, 0, 109S, 940E, 45, 996, TS, 34, NEQ, 65, 40, 45, 65, 1009, 340, 55, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011800, , BEST, 0, 112S, 939E, 45, 989, TS, 34, NEQ, 70, 70, 65, 80, 1008, 210, 55, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011806, , BEST, 0, 116S, 938E, 45, 997, TS, 34, NEQ, 65, 40, 45, 65, 1008, 220, 35, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011812, , BEST, 0, 119S, 937E, 45, 997, TS, 34, NEQ, 65, 40, 45, 65, 1008, 220, 35, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011818, , BEST, 0, 122S, 935E, 45, 993, TS, 34, NEQ, 70, 70, 65, 95, 1008, 225, 75, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011900, , BEST, 0, 124S, 933E, 55, 989, TS, 34, NEQ, 75, 115, 100, 85, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011900, , BEST, 0, 124S, 933E, 55, 989, TS, 50, NEQ, 30, 55, 50, 40, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024011906, , BEST, 0, 125S, 930E, 60, 988, TS, 34, NEQ, 75, 115, 100, 85, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011906, , BEST, 0, 125S, 930E, 60, 988, TS, 50, NEQ, 30, 55, 50, 40, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011912, , BEST, 0, 126S, 927E, 60, 988, TS, 34, NEQ, 75, 115, 100, 85, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011912, , BEST, 0, 126S, 927E, 60, 988, TS, 50, NEQ, 30, 55, 50, 40, 1008, 225, 35, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011918, , BEST, 0, 126S, 925E, 70, 988, TY, 34, NEQ, 90, 100, 95, 110, 1008, 180, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011918, , BEST, 0, 126S, 925E, 70, 988, TY, 50, NEQ, 50, 45, 45, 50, 1008, 180, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024011918, , BEST, 0, 126S, 925E, 70, 988, TY, 64, NEQ, 30, 30, 25, 25, 1008, 180, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012000, , BEST, 0, 124S, 928E, 75, 988, TY, 34, NEQ, 85, 75, 95, 115, 1008, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012000, , BEST, 0, 124S, 928E, 75, 988, TY, 50, NEQ, 55, 45, 45, 50, 1008, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012000, , BEST, 0, 124S, 928E, 75, 988, TY, 64, NEQ, 30, 30, 25, 25, 1008, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012006, , BEST, 0, 125S, 926E, 70, 988, TY, 34, NEQ, 85, 75, 95, 115, 1009, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012006, , BEST, 0, 125S, 926E, 70, 988, TY, 50, NEQ, 55, 45, 45, 50, 1009, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012006, , BEST, 0, 125S, 926E, 70, 988, TY, 64, NEQ, 30, 30, 25, 25, 1009, 180, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012012, , BEST, 0, 124S, 925E, 70, 980, TY, 34, NEQ, 75, 75, 80, 70, 1008, 230, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012012, , BEST, 0, 124S, 925E, 70, 980, TY, 50, NEQ, 40, 40, 40, 40, 1008, 230, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012012, , BEST, 0, 124S, 925E, 70, 980, TY, 64, NEQ, 25, 25, 25, 25, 1008, 230, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012018, , BEST, 0, 123S, 924E, 70, 981, TY, 34, NEQ, 75, 75, 80, 70, 1009, 200, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012018, , BEST, 0, 123S, 924E, 70, 981, TY, 50, NEQ, 40, 40, 40, 40, 1009, 200, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012018, , BEST, 0, 123S, 924E, 70, 981, TY, 64, NEQ, 25, 25, 25, 25, 1009, 200, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012100, , BEST, 0, 121S, 924E, 70, 984, TY, 34, NEQ, 65, 85, 85, 100, 1008, 150, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012100, , BEST, 0, 121S, 924E, 70, 984, TY, 50, NEQ, 40, 40, 40, 30, 1008, 150, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012100, , BEST, 0, 121S, 924E, 70, 984, TY, 64, NEQ, 25, 35, 30, 20, 1008, 150, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012106, , BEST, 0, 120S, 922E, 65, 984, TY, 34, NEQ, 65, 85, 85, 100, 1010, 225, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012106, , BEST, 0, 120S, 922E, 65, 984, TY, 50, NEQ, 40, 40, 40, 30, 1010, 225, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012106, , BEST, 0, 120S, 922E, 65, 984, TY, 64, NEQ, 25, 35, 30, 20, 1010, 225, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012112, , BEST, 0, 122S, 920E, 60, 985, TS, 34, NEQ, 80, 60, 80, 135, 1009, 265, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012112, , BEST, 0, 122S, 920E, 60, 985, TS, 50, NEQ, 35, 35, 35, 70, 1009, 265, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012118, , BEST, 0, 123S, 918E, 50, 994, TS, 34, NEQ, 80, 60, 80, 135, 1009, 180, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012118, , BEST, 0, 123S, 918E, 50, 994, TS, 50, NEQ, 35, 35, 35, 70, 1009, 180, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012200, , BEST, 0, 125S, 917E, 45, 991, TS, 34, NEQ, 80, 60, 80, 135, 1009, 170, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012206, , BEST, 0, 123S, 915E, 45, 999, TS, 34, NEQ, 80, 60, 80, 135, 1011, 270, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012212, , BEST, 0, 124S, 914E, 40, 1000, TS, 34, NEQ, 80, 60, 80, 135, 1010, 250, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012218, , BEST, 0, 126S, 915E, 40, 1000, TS, 34, NEQ, 50, 60, 50, 30, 1010, 250, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012300, , BEST, 0, 128S, 916E, 40, 1000, TS, 34, NEQ, 50, 60, 50, 30, 1010, 200, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012306, , BEST, 0, 130S, 916E, 45, 998, TS, 34, NEQ, 50, 50, 35, 0, 1010, 200, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012312, , BEST, 0, 133S, 916E, 50, 995, TS, 34, NEQ, 50, 50, 35, 0, 1010, 200, 30, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012318, , BEST, 0, 136S, 915E, 50, 994, TS, 34, NEQ, 50, 65, 60, 40, 1009, 180, 25, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012318, , BEST, 0, 136S, 915E, 50, 994, TS, 50, NEQ, 30, 30, 25, 15, 1009, 180, 25, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012400, , BEST, 0, 139S, 913E, 55, 993, TS, 34, NEQ, 60, 75, 70, 45, 1010, 165, 25, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012400, , BEST, 0, 139S, 913E, 55, 993, TS, 50, NEQ, 30, 35, 25, 15, 1010, 165, 25, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024012406, , BEST, 0, 140S, 912E, 65, 987, TY, 34, NEQ, 60, 75, 70, 45, 1010, 165, 15, 0, 20, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012406, , BEST, 0, 140S, 912E, 65, 987, TY, 50, NEQ, 30, 35, 25, 25, 1010, 165, 15, 0, 20, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012406, , BEST, 0, 140S, 912E, 65, 987, TY, 64, NEQ, 15, 15, 15, 15, 1010, 165, 15, 0, 20, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012412, , BEST, 0, 142S, 911E, 75, 980, TY, 34, NEQ, 60, 75, 70, 45, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012412, , BEST, 0, 142S, 911E, 75, 980, TY, 50, NEQ, 30, 35, 25, 25, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012412, , BEST, 0, 142S, 911E, 75, 980, TY, 64, NEQ, 15, 15, 15, 15, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012418, , BEST, 0, 145S, 911E, 80, 976, TY, 34, NEQ, 60, 75, 70, 45, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012418, , BEST, 0, 145S, 911E, 80, 976, TY, 50, NEQ, 30, 35, 25, 25, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012418, , BEST, 0, 145S, 911E, 80, 976, TY, 64, NEQ, 15, 15, 15, 15, 1010, 165, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012500, , BEST, 0, 147S, 908E, 80, 975, TY, 34, NEQ, 60, 75, 70, 45, 1009, 140, 15, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012500, , BEST, 0, 147S, 908E, 80, 975, TY, 50, NEQ, 30, 35, 25, 25, 1009, 140, 15, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012500, , BEST, 0, 147S, 908E, 80, 975, TY, 64, NEQ, 15, 15, 15, 15, 1009, 140, 15, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012506, , BEST, 0, 151S, 900E, 80, 978, TY, 34, NEQ, 60, 75, 125, 85, 1011, 170, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012506, , BEST, 0, 151S, 900E, 80, 978, TY, 50, NEQ, 40, 50, 45, 40, 1011, 170, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012506, , BEST, 0, 151S, 900E, 80, 978, TY, 64, NEQ, 30, 35, 30, 25, 1011, 170, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012512, , BEST, 0, 155S, 893E, 90, 974, TY, 34, NEQ, 70, 130, 115, 50, 1010, 150, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012512, , BEST, 0, 155S, 893E, 90, 974, TY, 50, NEQ, 35, 60, 55, 35, 1010, 150, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012512, , BEST, 0, 155S, 893E, 90, 974, TY, 64, NEQ, 25, 40, 40, 25, 1010, 150, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012518, , BEST, 0, 160S, 883E, 95, 972, TY, 34, NEQ, 70, 130, 115, 50, 1011, 120, 20, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012518, , BEST, 0, 160S, 883E, 95, 972, TY, 50, NEQ, 35, 60, 55, 35, 1011, 120, 20, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012518, , BEST, 0, 160S, 883E, 95, 972, TY, 64, NEQ, 25, 40, 40, 30, 1011, 120, 20, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012600, , BEST, 0, 163S, 875E, 105, 958, TY, 34, NEQ, 60, 130, 110, 55, 1009, 105, 15, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012600, , BEST, 0, 163S, 875E, 105, 958, TY, 50, NEQ, 35, 50, 45, 35, 1009, 105, 15, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012600, , BEST, 0, 163S, 875E, 105, 958, TY, 64, NEQ, 30, 30, 30, 25, 1009, 105, 15, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012606, , BEST, 0, 169S, 865E, 105, 958, TY, 34, NEQ, 60, 130, 110, 55, 1011, 150, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012606, , BEST, 0, 169S, 865E, 105, 958, TY, 50, NEQ, 35, 50, 45, 35, 1011, 150, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012606, , BEST, 0, 169S, 865E, 105, 958, TY, 64, NEQ, 30, 30, 30, 25, 1011, 150, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012612, , BEST, 0, 175S, 853E, 110, 958, TY, 34, NEQ, 65, 95, 90, 45, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012612, , BEST, 0, 175S, 853E, 110, 958, TY, 50, NEQ, 40, 45, 40, 30, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012612, , BEST, 0, 175S, 853E, 110, 958, TY, 64, NEQ, 30, 35, 30, 20, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012618, , BEST, 0, 182S, 842E, 105, 960, TY, 34, NEQ, 65, 95, 90, 45, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012618, , BEST, 0, 182S, 842E, 105, 960, TY, 50, NEQ, 40, 45, 40, 30, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012618, , BEST, 0, 182S, 842E, 105, 960, TY, 64, NEQ, 30, 35, 30, 20, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012700, , BEST, 0, 187S, 831E, 100, 964, TY, 34, NEQ, 65, 95, 90, 45, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012700, , BEST, 0, 187S, 831E, 100, 964, TY, 50, NEQ, 40, 45, 40, 30, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012700, , BEST, 0, 187S, 831E, 100, 964, TY, 64, NEQ, 30, 35, 30, 20, 1010, 120, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012706, , BEST, 0, 190S, 820E, 100, 964, TY, 34, NEQ, 45, 55, 50, 50, 1010, 140, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012706, , BEST, 0, 190S, 820E, 100, 964, TY, 50, NEQ, 30, 30, 35, 25, 1010, 140, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012706, , BEST, 0, 190S, 820E, 100, 964, TY, 64, NEQ, 25, 25, 30, 20, 1010, 140, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012712, , BEST, 0, 193S, 806E, 100, 964, TY, 34, NEQ, 45, 55, 50, 50, 1010, 160, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012712, , BEST, 0, 193S, 806E, 100, 964, TY, 50, NEQ, 30, 30, 35, 25, 1010, 160, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012712, , BEST, 0, 193S, 806E, 100, 964, TY, 64, NEQ, 25, 25, 30, 20, 1010, 160, 12, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012718, , BEST, 0, 195S, 794E, 100, 965, TY, 34, NEQ, 55, 80, 80, 55, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012718, , BEST, 0, 195S, 794E, 100, 965, TY, 50, NEQ, 20, 40, 40, 20, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012718, , BEST, 0, 195S, 794E, 100, 965, TY, 64, NEQ, 10, 20, 20, 10, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012800, , BEST, 0, 200S, 779E, 95, 968, TY, 34, NEQ, 55, 80, 80, 55, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012800, , BEST, 0, 200S, 779E, 95, 968, TY, 50, NEQ, 20, 40, 40, 20, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012800, , BEST, 0, 200S, 779E, 95, 968, TY, 64, NEQ, 10, 20, 20, 10, 1012, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012806, , BEST, 0, 204S, 764E, 95, 970, TY, 34, NEQ, 55, 90, 60, 45, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012806, , BEST, 0, 204S, 764E, 95, 970, TY, 50, NEQ, 35, 35, 30, 20, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012806, , BEST, 0, 204S, 764E, 95, 970, TY, 64, NEQ, 20, 20, 20, 15, 1010, 120, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012812, , BEST, 0, 210S, 748E, 100, 970, TY, 34, NEQ, 60, 85, 65, 45, 1008, 105, 10, 0, 10, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012812, , BEST, 0, 210S, 748E, 100, 970, TY, 50, NEQ, 30, 30, 30, 30, 1008, 105, 10, 0, 10, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012812, , BEST, 0, 210S, 748E, 100, 970, TY, 64, NEQ, 20, 20, 20, 20, 1008, 105, 10, 0, 10, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012818, , BEST, 0, 217S, 732E, 110, 955, TY, 34, NEQ, 60, 85, 70, 45, 1010, 110, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012818, , BEST, 0, 217S, 732E, 110, 955, TY, 50, NEQ, 40, 40, 30, 30, 1010, 110, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012818, , BEST, 0, 217S, 732E, 110, 955, TY, 64, NEQ, 20, 20, 15, 15, 1010, 110, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012900, , BEST, 0, 227S, 719E, 115, 954, TY, 34, NEQ, 60, 75, 60, 60, 1010, 115, 10, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012900, , BEST, 0, 227S, 719E, 115, 954, TY, 50, NEQ, 40, 45, 45, 35, 1010, 115, 10, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012900, , BEST, 0, 227S, 719E, 115, 954, TY, 64, NEQ, 30, 40, 35, 30, 1010, 115, 10, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012906, , BEST, 0, 238S, 711E, 115, 948, TY, 34, NEQ, 60, 75, 60, 60, 1010, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012906, , BEST, 0, 238S, 711E, 115, 948, TY, 50, NEQ, 40, 45, 45, 35, 1010, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012906, , BEST, 0, 238S, 711E, 115, 948, TY, 64, NEQ, 30, 40, 35, 30, 1010, 140, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012912, , BEST, 0, 251S, 708E, 120, 960, TY, 34, NEQ, 60, 75, 60, 60, 1009, 160, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012912, , BEST, 0, 251S, 708E, 120, 960, TY, 50, NEQ, 40, 45, 45, 35, 1009, 160, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012912, , BEST, 0, 251S, 708E, 120, 960, TY, 64, NEQ, 30, 40, 35, 30, 1009, 160, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012918, , BEST, 0, 264S, 709E, 110, 950, TY, 34, NEQ, 60, 75, 60, 60, 1010, 125, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012918, , BEST, 0, 264S, 709E, 110, 950, TY, 50, NEQ, 40, 45, 45, 35, 1010, 125, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024012918, , BEST, 0, 264S, 709E, 110, 950, TY, 64, NEQ, 30, 40, 35, 30, 1010, 125, 12, 0, 15, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013000, , BEST, 0, 280S, 717E, 105, 952, TY, 34, NEQ, 65, 65, 65, 65, 1009, 130, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013000, , BEST, 0, 280S, 717E, 105, 952, TY, 50, NEQ, 40, 45, 45, 40, 1009, 130, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013000, , BEST, 0, 280S, 717E, 105, 952, TY, 64, NEQ, 35, 40, 35, 35, 1009, 130, 17, 0, 25, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013006, , BEST, 0, 291S, 728E, 95, 961, TY, 34, NEQ, 65, 65, 65, 65, 1010, 125, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013006, , BEST, 0, 291S, 728E, 95, 961, TY, 50, NEQ, 40, 45, 45, 40, 1010, 125, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013006, , BEST, 0, 291S, 728E, 95, 961, TY, 64, NEQ, 35, 40, 35, 35, 1010, 125, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013012, , BEST, 0, 300S, 745E, 90, 965, TY, 34, NEQ, 80, 85, 70, 65, 1010, 140, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013012, , BEST, 0, 300S, 745E, 90, 965, TY, 50, NEQ, 40, 50, 45, 40, 1010, 140, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013012, , BEST, 0, 300S, 745E, 90, 965, TY, 64, NEQ, 25, 30, 25, 30, 1010, 140, 20, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013018, , BEST, 0, 308S, 762E, 90, 965, TY, 34, NEQ, 80, 85, 70, 65, 1010, 140, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013018, , BEST, 0, 308S, 762E, 90, 965, TY, 50, NEQ, 40, 50, 45, 40, 1010, 140, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013018, , BEST, 0, 308S, 762E, 90, 965, TY, 64, NEQ, 25, 30, 25, 30, 1010, 140, 30, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013100, , BEST, 0, 314S, 789E, 80, 972, TY, 34, NEQ, 100, 75, 110, 95, 1012, 190, 25, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013100, , BEST, 0, 314S, 789E, 80, 972, TY, 50, NEQ, 45, 30, 55, 45, 1012, 190, 25, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013100, , BEST, 0, 314S, 789E, 80, 972, TY, 64, NEQ, 25, 15, 25, 20, 1012, 190, 25, 0, 0, S, 0, , 0, 0, ANGGREK, D, +SH, 06, 2024013106, , BEST, 0, 324S, 828E, 65, 983, TY, 34, NEQ, 110, 120, 120, 110, 1010, 230, 40, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024013106, , BEST, 0, 324S, 828E, 65, 983, TY, 50, NEQ, 80, 85, 70, 65, 1010, 230, 40, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024013106, , BEST, 0, 324S, 828E, 65, 983, TY, 64, NEQ, 50, 60, 0, 0, 1010, 230, 40, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024013112, , BEST, 0, 342S, 875E, 55, 986, TS, 34, NEQ, 110, 120, 120, 110, 1006, 200, 40, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024013112, , BEST, 0, 342S, 875E, 55, 986, TS, 50, NEQ, 80, 85, 70, 65, 1006, 200, 40, 0, 0, S, 0, , 0, 0, ANGGREK, M, +SH, 06, 2024013118, , BEST, 0, 361S, 915E, 55, 986, TS, 34, NEQ, 110, 120, 120, 110, 1006, 200, 40, 0, 0, S, 0, , 0, 0, ANGGREK, , +SH, 06, 2024013118, , BEST, 0, 361S, 915E, 55, 986, TS, 50, NEQ, 80, 85, 70, 65, 1006, 200, 40, 0, 0, S, 0, , 0, 0, ANGGREK, , diff --git a/tests/test_all.sh b/tests/test_all.sh index 0b587433d..0cf622c70 100755 --- a/tests/test_all.sh +++ b/tests/test_all.sh @@ -41,6 +41,8 @@ for call in \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ascat_noaa_25km.tc.windbarbs.imagery_windbarbs.sh" \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ascat_noaa_50km.tc.wind-ambiguities.imagery_windbarbs.sh" \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ascat_uhr.tc.wind-ambiguities.imagery_windbarbs.sh" \ + "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ascat_uhr.tc.windbarbs.imagery_windbarbs.sh" \ + "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ascat_uhr.tc.windspeed.imagery_clean.sh" \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/atms.tc.165H.netcdf_geoips.sh" \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/cli_dummy_script.sh" \ "$GEOIPS_PACKAGES_DIR/geoips/tests/scripts/ewsg.static.Infrared.imagery_clean.sh" \ diff --git a/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.json b/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.json new file mode 100644 index 000000000..4d679fb84 --- /dev/null +++ b/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.json @@ -0,0 +1,193 @@ +{ + "name": "I_have_been_modified_more_recently_than_my_parent_yaml", + "instructions": { + "config": { + "help_str": "Various configuration-based commands for setting up your geoips environment.\nCurrently supports `geoips config install ` command.\n", + "usage_str": "To use, type `geoips config ...`\n", + "output_info": [ + "Output related to config process that was run." + ] + }, + "config_install": { + "help_str": "Install the appropriate test dataset and/or package based on the arguments\nprovided. To see a list of available test datasets for install, run\n`geoips list test-datasets`.\n", + "usage_str": "To use, type `geoips config install `.\n", + "output_info": [ + "Not Applicable" + ] + }, + "get": { + "help_str": "Retrieve an appropriate GeoIPS artifact. Currently supported `get` calls are:\n[\"family\", \"interface\", \"package\", \"plugin\"]\n", + "usage_str": "To use, type `geoips get ...`.\n", + "output_info": [ + "Information related to the artifact that was retrieved." + ] + }, + "get_family": { + "help_str": "Retrieve the appropriate GeoIPS Family alongside descriptive information of\nthat family. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS families, run:\n`geoips list interfaces`.\n", + "usage_str": "To use, type `geoips get family `, where\n is a valid GeoIPS Interface and is a supported\nfamily member of that interface.\n", + "output_info": [ + "Docstring", + "Family Name", + "Family Path", + "Interface Name", + "Interface Type", + "Required Args / Schema" + ] + }, + "get_interface": { + "help_str": "Retrieve the appropriate GeoIPS Interface alongside descriptive information of\nthat interface. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS interfaces, run:\n`geoips list interfaces`.\n", + "usage_str": "To use, type `geoips get interface `, where is a\nvalid GeoIPS Interface.\n", + "output_info": [ + "Absolute Path", + "Docstring", + "Documentation Link (If applicable)", + "Interface Name", + "Interface Type", + "Supported Families" + ] + }, + "get_package": { + "help_str": "Retrieve the appropriate GeoIPS Package alongside descriptive information of\nthat package. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS packages, run:\n`geoips list packages`.\n", + "usage_str": "To use, type `geoips get package `, where is a\nvalid GeoIPS Package.\n", + "output_info": [ + "GeoIPS Package", + "Docstring", + "Package Path", + "Documentation Link" + ] + }, + "get_plugin": { + "help_str": "Retrieve the appropriate GeoIPS Plugin alongside descriptive information of\nthat plugin. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS Plugins, run:\n`geoips list plugins`.\n", + "usage_str": "To use, type `geoips get plugin `, where\n is a valid GeoIPS Interface and is a valid plugin\nidentifier that has been implemented in any installed GeoIPS package.\n", + "output_info": [ + "Docstring", + "Documentation Link (If applicable)", + "Family", + "Interface Name", + "Package", + "Relative Path", + "Plugin Specific Info (signature / source_names / available overrides)" + ] + }, + "list": { + "help_str": "List off available GeoIPS artifacts provided further commands. Current artifacts\navailable for listing are: [\"interface\", \"interfaces\", \"packages\", \"plugins\",\n\"scripts\", \"test-datasets\", \"unit-tests\"].\n", + "usage_str": "To use, type `geoips list `.\n", + "output_info": [ + "Additional info related to the artifact[s] being listed." + ] + }, + "list_interface": { + "help_str": "List off plugins found under a certain interface. This is useful for seeing what\nplugins have been implemented through all, or a certain, GeoIPS Package. This\ninformation can then be used with\n`geoips get-plugin ` to get more information about a\ncertain plugin.\n", + "usage_str": "To use, type `geoips list interface `. Optionally includes `-p`\nflag for seeing what plugins are in a certain package (-p). For example, to see\nwhat plugins under a certain interface have been created in a single package,\nrun `geoips list interface -p `, or you can remove the\n`-p` flag to see what's been created in every package.\n", + "output_info": [ + "Family", + "Interface Name", + "Interface Type", + "Package", + "Plugin Name", + "Relative Path" + ] + }, + "list_interfaces": { + "help_str": "List off every GeoIPS Interface. This is useful for seeing what is\navailable throughout GeoIPS, but can also be extended to see what interfaces have\nbeen implemented in other geoips package, or just a certain package. This\ninformation can then be used with `geoips get-interface ` to get\nmore information about a certain interface.\n", + "usage_str": "To use, type `geoips list interfaces`. Optionally includes `-i` and `-p` flags for\nseeing what has been implemented (-i) in a certain package (-p), or all packages.\nFor example, to see what interfaces have been implemented in a certain package,\nrun `geoips list interfaces -i -p `, or you can remove the `-p` flag to\nsee what's been implemented in every package.\n", + "output_info": [ + "Absolute Path", + "Docstring", + "Documentation Link (if applicable)", + "Interface Name", + "Interface Type", + "Package", + "Supported Families" + ] + }, + "list_plugins": { + "help_str": "List off plugins found under all, or a specified GeoIPS package. This information\ncan then be used with `geoips get-plugin ` to get\nmore information about a certain plugin.\n", + "usage_str": "To use, type `geoips list plugins`. Optionally includes `-p` flag for\nseeing what plugins are in a certain package (-p). For example, to see what\nplugins have been created in a single package, run\n`geoips list plugins -p `, or you can remove the `-p` flag to see what's\nbeen created in every package.\n", + "output_info": [ + "Family", + "Interface Name", + "Interface Type", + "Package", + "Plugin Name", + "Relative Path" + ] + }, + "list_packages": { + "help_str": "List off packages found under the GeoIPS Namespace, alongside their path and\ntop-level docstring. This command will list every GeoIPS Package found, provided\nit has already been installed.\n", + "usage_str": "To use, type `geoips list packages`.\n", + "output_info": [ + "Docstring", + "Package", + "Relative Path" + ] + }, + "list_scripts": { + "help_str": "List off scripts found under all, or a specified GeoIPS package. This information\ncan then be used with `geoips run ` to run a certain\nprocess workflow, defined in a bash script.\n", + "usage_str": "To use, type `geoips list scripts`. Optionally includes `-p` flag for\nseeing what scripts are in a certain package (-p). For example, to see what\nscripts have been created in a single package, run\n`geoips list scripts -p `, or you can remove the `-p` flag to see what's\nbeen created in every package.\n", + "output_info": [ + "Package", + "Script Name" + ] + }, + "list_test-datasets": { + "help_str": "List off test-datasets used for testing GeoIPS and other GeoIPS Packages.\nThis data is used for running tests via `geoips run `.\n", + "usage_str": "To use, type `geoips list test-datasets`.\n", + "output_info": [ + "Data Host", + "Dataset Name" + ] + }, + "list_unit-tests": { + "help_str": "List off unit-tests used in a certain GeoIPS Package.\nThis data can be used to run unit tests via `geoips test unit-test ...`.\n", + "usage_str": "To use, type `geoips list unit-tests -p `.\n", + "output_info": [ + "GeoIPS Package", + "Unit Test Directory", + "Unit Test Name" + ] + }, + "run": { + "help_str": "Run a geoips process workflow (procflow), given a GeoIPS package and a script\nname. This will do the exact same thing as being in the directory of a certain\nscript, and running it via `./`. To see what scripts are available,\nyou can run `geoips list-scripts <-p>`, where `-p` can be used alongside a GeoIPS\nPackage name if you want scripts from a single package.\n", + "usage_str": "To use, type `geoips run -p `.\n", + "output_info": [ + "Log Output of the Process Workflow (Procflow)" + ] + }, + "test": { + "help_str": "Run tests implemented in GeoIPS or a separate GeoIPS Package. This could be\nrunning a set of linters on a specific package, running a certain integration or\nnormal test script, or running unit tests. Current options are [\"linting\",\n\"script\", \"unit-test\"].\n", + "usage_str": "To use, type `geoips test `.\n", + "output_info": [ + "Output of the Test Being Ran" + ] + }, + "test_linting": { + "help_str": "Run all GeoIPS linters on a specific GeoIPS Package. This will test whether or not\nthe code you wrote adheres to the specified coding conventions set up by GeoIPS.\nDefaults to the 'geoips' package.\n", + "usage_str": "To use, type `geoips test linting -p `.\n", + "output_info": [ + "Output of the Linters" + ] + }, + "test_script": { + "help_str": "Runs a specific test script (integration-based or normal), found within a certain\nGeoIPS package. If this is specified as an integration test, this will only work\nusing the 'geoips' package. Use '--integration' to specify that the script\nis integration-based. Defaults to the 'geoips' package.\n", + "usage_str": "To use, type `geoips test script -p <--integration> `.\n", + "output_info": [ + "Output of the Script Being Ran" + ] + }, + "test_unit-test": { + "help_str": "Run tests unit-test[s] implemented in a certain GeoIPS package. This command\nexpects that the unit tests implemented are pytest-based. Defaults to the 'geoips'\npackage.\n", + "usage_str": "To use, type\n`geoips test unit-test -p <-n> `\n", + "output_info": [ + "Output of the Unit Test Being Ran" + ] + }, + "validate": { + "help_str": "Validate a GeoIPS Plugin found at . While this is done under the hood via\nGeoIPS, this is an easy way to test whether or not the plugin you are developing\nis valid.\n", + "usage_str": "To use, type `geoips validate `. Do it man. NOW.\n", + "output_info": [ + "A message telling you if the plugin is valid or not and why." + ] + } + } +} \ No newline at end of file diff --git a/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.yaml b/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.yaml new file mode 100644 index 000000000..7e3486a27 --- /dev/null +++ b/tests/unit_tests/commandline/cmd_instructions/json_newer/cmd_instructions.yaml @@ -0,0 +1,263 @@ +name: geoips_cmd_instructions +instructions: + config: + help_str: | + Various configuration-based commands for setting up your geoips environment. + Currently supports `geoips config install ` command. + usage_str: | + To use, type `geoips config ...` + output_info: + - Output related to config process that was run. + config_install: + help_str: | + Install the appropriate test dataset and/or package based on the arguments + provided. To see a list of available test datasets for install, run + `geoips list test-datasets`. + usage_str: | + To use, type `geoips config install `. + output_info: + - Not Applicable + get: + help_str: | + Retrieve an appropriate GeoIPS artifact. Currently supported `get` calls are: + ["family", "interface", "package", "plugin"] + usage_str: | + To use, type `geoips get ...`. + output_info: + - Information related to the artifact that was retrieved. + get_family: + help_str: | + Retrieve the appropriate GeoIPS Family alongside descriptive information of + that family. See output_info for each datum provided when this command is + called. For a listing of available GeoIPS families, run: + `geoips list interfaces`. + usage_str: | + To use, type `geoips get family `, where + is a valid GeoIPS Interface and is a supported + family member of that interface. + output_info: + - Docstring + - Family Name + - Family Path + - Interface Name + - Interface Type + - Required Args / Schema + get_interface: + help_str: | + Retrieve the appropriate GeoIPS Interface alongside descriptive information of + that interface. See output_info for each datum provided when this command is + called. For a listing of available GeoIPS interfaces, run: + `geoips list interfaces`. + usage_str: | + To use, type `geoips get interface `, where is a + valid GeoIPS Interface. + output_info: + - Absolute Path + - Docstring + - Documentation Link (If applicable) + - Interface Name + - Interface Type + - Supported Families + get_package: + help_str: | + Retrieve the appropriate GeoIPS Package alongside descriptive information of + that package. See output_info for each datum provided when this command is + called. For a listing of available GeoIPS packages, run: + `geoips list packages`. + usage_str: | + To use, type `geoips get package `, where is a + valid GeoIPS Package. + output_info: + - GeoIPS Package + - Docstring + - Package Path + - Documentation Link + get_plugin: + help_str: | + Retrieve the appropriate GeoIPS Plugin alongside descriptive information of + that plugin. See output_info for each datum provided when this command is + called. For a listing of available GeoIPS Plugins, run: + `geoips list plugins`. + usage_str: | + To use, type `geoips get plugin `, where + is a valid GeoIPS Interface and is a valid plugin + identifier that has been implemented in any installed GeoIPS package. + output_info: + - Docstring + - Documentation Link (If applicable) + - Family + - Interface Name + - Package + - Relative Path + - Plugin Specific Info (signature / source_names / available overrides) + list: + help_str: | + List off available GeoIPS artifacts provided further commands. Current artifacts + available for listing are: ["interface", "interfaces", "packages", "plugins", + "scripts", "test-datasets", "unit-tests"]. + usage_str: | + To use, type `geoips list `. + output_info: + - Additional info related to the artifact[s] being listed. + list_interface: + help_str: | + List off plugins found under a certain interface. This is useful for seeing what + plugins have been implemented through all, or a certain, GeoIPS Package. This + information can then be used with + `geoips get-plugin ` to get more information about a + certain plugin. + usage_str: | + To use, type `geoips list interface `. Optionally includes `-p` + flag for seeing what plugins are in a certain package (-p). For example, to see + what plugins under a certain interface have been created in a single package, + run `geoips list interface -p `, or you can remove the + `-p` flag to see what's been created in every package. + output_info: + - Family + - Interface Name + - Interface Type + - Package + - Plugin Name + - Relative Path + list_interfaces: + help_str: | + List off every GeoIPS Interface. This is useful for seeing what is + available throughout GeoIPS, but can also be extended to see what interfaces have + been implemented in other geoips package, or just a certain package. This + information can then be used with `geoips get-interface ` to get + more information about a certain interface. + usage_str: | + To use, type `geoips list interfaces`. Optionally includes `-i` and `-p` flags for + seeing what has been implemented (-i) in a certain package (-p), or all packages. + For example, to see what interfaces have been implemented in a certain package, + run `geoips list interfaces -i -p `, or you can remove the `-p` flag to + see what's been implemented in every package. + output_info: + - Absolute Path + - Docstring + - Documentation Link (if applicable) + - Interface Name + - Interface Type + - Package + - Supported Families + list_plugins: + help_str: | + List off plugins found under all, or a specified GeoIPS package. This information + can then be used with `geoips get-plugin ` to get + more information about a certain plugin. + usage_str: | + To use, type `geoips list plugins`. Optionally includes `-p` flag for + seeing what plugins are in a certain package (-p). For example, to see what + plugins have been created in a single package, run + `geoips list plugins -p `, or you can remove the `-p` flag to see what's + been created in every package. + output_info: + - Family + - Interface Name + - Interface Type + - Package + - Plugin Name + - Relative Path + list_packages: + help_str: | + List off packages found under the GeoIPS Namespace, alongside their path and + top-level docstring. This command will list every GeoIPS Package found, provided + it has already been installed. + usage_str: | + To use, type `geoips list packages`. + output_info: + - Docstring + - Package + - Relative Path + list_scripts: + help_str: | + List off scripts found under all, or a specified GeoIPS package. This information + can then be used with `geoips run ` to run a certain + process workflow, defined in a bash script. + usage_str: | + To use, type `geoips list scripts`. Optionally includes `-p` flag for + seeing what scripts are in a certain package (-p). For example, to see what + scripts have been created in a single package, run + `geoips list scripts -p `, or you can remove the `-p` flag to see what's + been created in every package. + output_info: + - Package + - Script Name + list_test-datasets: + help_str: | + List off test-datasets used for testing GeoIPS and other GeoIPS Packages. + This data is used for running tests via `geoips run `. + usage_str: | + To use, type `geoips list test-datasets`. + output_info: + - Data Host + - Dataset Name + list_unit-tests: + help_str: | + List off unit-tests used in a certain GeoIPS Package. + This data can be used to run unit tests via `geoips test unit-test ...`. + usage_str: | + To use, type `geoips list unit-tests -p `. + output_info: + - GeoIPS Package + - Unit Test Directory + - Unit Test Name + run: + help_str: | + Run a geoips process workflow (procflow), given a GeoIPS package and a script + name. This will do the exact same thing as being in the directory of a certain + script, and running it via `./`. To see what scripts are available, + you can run `geoips list-scripts <-p>`, where `-p` can be used alongside a GeoIPS + Package name if you want scripts from a single package. + usage_str: | + To use, type `geoips run -p `. + output_info: + - Log Output of the Process Workflow (Procflow) + test: + help_str: | + Run tests implemented in GeoIPS or a separate GeoIPS Package. This could be + running a set of linters on a specific package, running a certain integration or + normal test script, or running unit tests. Current options are ["linting", + "script", "unit-test"]. + usage_str: | + To use, type `geoips test `. + output_info: + - Output of the Test Being Ran + test_linting: + help_str: | + Run all GeoIPS linters on a specific GeoIPS Package. This will test whether or not + the code you wrote adheres to the specified coding conventions set up by GeoIPS. + Defaults to the 'geoips' package. + usage_str: | + To use, type `geoips test linting -p `. + output_info: + - Output of the Linters + test_script: + help_str: | + Runs a specific test script (integration-based or normal), found within a certain + GeoIPS package. If this is specified as an integration test, this will only work + using the 'geoips' package. Use '--integration' to specify that the script + is integration-based. Defaults to the 'geoips' package. + usage_str: | + To use, type `geoips test script -p <--integration> `. + output_info: + - Output of the Script Being Ran + test_unit-test: + help_str: | + Run tests unit-test[s] implemented in a certain GeoIPS package. This command + expects that the unit tests implemented are pytest-based. Defaults to the 'geoips' + package. + usage_str: | + To use, type + `geoips test unit-test -p <-n> ` + output_info: + - Output of the Unit Test Being Ran + validate: + help_str: | + Validate a GeoIPS Plugin found at . While this is done under the hood via + GeoIPS, this is an easy way to test whether or not the plugin you are developing + is valid. + usage_str: | + To use, type `geoips validate `. Do it man. NOW. + output_info: + - A message telling you if the plugin is valid or not and why. diff --git a/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.json b/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.json new file mode 100644 index 000000000..bd8083ab3 --- /dev/null +++ b/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.json @@ -0,0 +1,193 @@ +{ + "name": "geoips_cmd_instructions", + "instructions": { + "config": { + "help_str": "Various configuration-based commands for setting up your geoips environment.\nCurrently supports `geoips config install ` command.\n", + "usage_str": "To use, type `geoips config ...`\n", + "output_info": [ + "Output related to config process that was run." + ] + }, + "config_install": { + "help_str": "Install the appropriate test dataset and/or package based on the arguments\nprovided. To see a list of available test datasets for install, run\n`geoips list test-datasets`.\n", + "usage_str": "To use, type `geoips config install `.\n", + "output_info": [ + "Not Applicable" + ] + }, + "get": { + "help_str": "Retrieve an appropriate GeoIPS artifact. Currently supported `get` calls are:\n[\"family\", \"interface\", \"package\", \"plugin\"]\n", + "usage_str": "To use, type `geoips get ...`.\n", + "output_info": [ + "Information related to the artifact that was retrieved." + ] + }, + "get_family": { + "help_str": "Retrieve the appropriate GeoIPS Family alongside descriptive information of\nthat family. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS families, run:\n`geoips list interfaces`.\n", + "usage_str": "To use, type `geoips get family `, where\n is a valid GeoIPS Interface and is a supported\nfamily member of that interface.\n", + "output_info": [ + "Docstring", + "Family Name", + "Family Path", + "Interface Name", + "Interface Type", + "Required Args / Schema" + ] + }, + "get_interface": { + "help_str": "Retrieve the appropriate GeoIPS Interface alongside descriptive information of\nthat interface. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS interfaces, run:\n`geoips list interfaces`.\n", + "usage_str": "To use, type `geoips get interface `, where is a\nvalid GeoIPS Interface.\n", + "output_info": [ + "Absolute Path", + "Docstring", + "Documentation Link (If applicable)", + "Interface Name", + "Interface Type", + "Supported Families" + ] + }, + "get_package": { + "help_str": "Retrieve the appropriate GeoIPS Package alongside descriptive information of\nthat package. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS packages, run:\n`geoips list packages`.\n", + "usage_str": "To use, type `geoips get package `, where is a\nvalid GeoIPS Package.\n", + "output_info": [ + "GeoIPS Package", + "Docstring", + "Package Path", + "Documentation Link" + ] + }, + "get_plugin": { + "help_str": "Retrieve the appropriate GeoIPS Plugin alongside descriptive information of\nthat plugin. See output_info for each datum provided when this command is\ncalled. For a listing of available GeoIPS Plugins, run:\n`geoips list plugins`.\n", + "usage_str": "To use, type `geoips get plugin `, where\n is a valid GeoIPS Interface and is a valid plugin\nidentifier that has been implemented in any installed GeoIPS package.\n", + "output_info": [ + "Docstring", + "Documentation Link (If applicable)", + "Family", + "Interface Name", + "Package", + "Relative Path", + "Plugin Specific Info (signature / source_names / available overrides)" + ] + }, + "list": { + "help_str": "List off available GeoIPS artifacts provided further commands. Current artifacts\navailable for listing are: [\"interface\", \"interfaces\", \"packages\", \"plugins\",\n\"scripts\", \"test-datasets\", \"unit-tests\"].\n", + "usage_str": "To use, type `geoips list `.\n", + "output_info": [ + "Additional info related to the artifact[s] being listed." + ] + }, + "list_interface": { + "help_str": "List off plugins found under a certain interface. This is useful for seeing what\nplugins have been implemented through all, or a certain, GeoIPS Package. This\ninformation can then be used with\n`geoips get-plugin ` to get more information about a\ncertain plugin.\n", + "usage_str": "To use, type `geoips list interface `. Optionally includes `-p`\nflag for seeing what plugins are in a certain package (-p). For example, to see\nwhat plugins under a certain interface have been created in a single package,\nrun `geoips list interface -p `, or you can remove the\n`-p` flag to see what's been created in every package.\n", + "output_info": [ + "Family", + "Interface Name", + "Interface Type", + "Package", + "Plugin Name", + "Relative Path" + ] + }, + "list_interfaces": { + "help_str": "List off every GeoIPS Interface. This is useful for seeing what is\navailable throughout GeoIPS, but can also be extended to see what interfaces have\nbeen implemented in other geoips package, or just a certain package. This\ninformation can then be used with `geoips get-interface ` to get\nmore information about a certain interface.\n", + "usage_str": "To use, type `geoips list interfaces`. Optionally includes `-i` and `-p` flags for\nseeing what has been implemented (-i) in a certain package (-p), or all packages.\nFor example, to see what interfaces have been implemented in a certain package,\nrun `geoips list interfaces -i -p `, or you can remove the `-p` flag to\nsee what's been implemented in every package.\n", + "output_info": [ + "Absolute Path", + "Docstring", + "Documentation Link (if applicable)", + "Interface Name", + "Interface Type", + "Package", + "Supported Families" + ] + }, + "list_plugins": { + "help_str": "List off plugins found under all, or a specified GeoIPS package. This information\ncan then be used with `geoips get-plugin ` to get\nmore information about a certain plugin.\n", + "usage_str": "To use, type `geoips list plugins`. Optionally includes `-p` flag for\nseeing what plugins are in a certain package (-p). For example, to see what\nplugins have been created in a single package, run\n`geoips list plugins -p `, or you can remove the `-p` flag to see what's\nbeen created in every package.\n", + "output_info": [ + "Family", + "Interface Name", + "Interface Type", + "Package", + "Plugin Name", + "Relative Path" + ] + }, + "list_packages": { + "help_str": "List off packages found under the GeoIPS Namespace, alongside their path and\ntop-level docstring. This command will list every GeoIPS Package found, provided\nit has already been installed.\n", + "usage_str": "To use, type `geoips list packages`.\n", + "output_info": [ + "Docstring", + "Package", + "Relative Path" + ] + }, + "list_scripts": { + "help_str": "List off scripts found under all, or a specified GeoIPS package. This information\ncan then be used with `geoips run ` to run a certain\nprocess workflow, defined in a bash script.\n", + "usage_str": "To use, type `geoips list scripts`. Optionally includes `-p` flag for\nseeing what scripts are in a certain package (-p). For example, to see what\nscripts have been created in a single package, run\n`geoips list scripts -p `, or you can remove the `-p` flag to see what's\nbeen created in every package.\n", + "output_info": [ + "Package", + "Script Name" + ] + }, + "list_test-datasets": { + "help_str": "List off test-datasets used for testing GeoIPS and other GeoIPS Packages.\nThis data is used for running tests via `geoips run `.\n", + "usage_str": "To use, type `geoips list test-datasets`.\n", + "output_info": [ + "Data Host", + "Dataset Name" + ] + }, + "list_unit-tests": { + "help_str": "List off unit-tests used in a certain GeoIPS Package.\nThis data can be used to run unit tests via `geoips test unit-test ...`.\n", + "usage_str": "To use, type `geoips list unit-tests -p `.\n", + "output_info": [ + "GeoIPS Package", + "Unit Test Directory", + "Unit Test Name" + ] + }, + "run": { + "help_str": "Run a geoips process workflow (procflow), given a GeoIPS package and a script\nname. This will do the exact same thing as being in the directory of a certain\nscript, and running it via `./`. To see what scripts are available,\nyou can run `geoips list-scripts <-p>`, where `-p` can be used alongside a GeoIPS\nPackage name if you want scripts from a single package.\n", + "usage_str": "To use, type `geoips run -p `.\n", + "output_info": [ + "Log Output of the Process Workflow (Procflow)" + ] + }, + "test": { + "help_str": "Run tests implemented in GeoIPS or a separate GeoIPS Package. This could be\nrunning a set of linters on a specific package, running a certain integration or\nnormal test script, or running unit tests. Current options are [\"linting\",\n\"script\", \"unit-test\"].\n", + "usage_str": "To use, type `geoips test `.\n", + "output_info": [ + "Output of the Test Being Ran" + ] + }, + "test_linting": { + "help_str": "Run all GeoIPS linters on a specific GeoIPS Package. This will test whether or not\nthe code you wrote adheres to the specified coding conventions set up by GeoIPS.\nDefaults to the 'geoips' package.\n", + "usage_str": "To use, type `geoips test linting -p `.\n", + "output_info": [ + "Output of the Linters" + ] + }, + "test_script": { + "help_str": "Runs a specific test script (integration-based or normal), found within a certain\nGeoIPS package. If this is specified as an integration test, this will only work\nusing the 'geoips' package. Use '--integration' to specify that the script\nis integration-based. Defaults to the 'geoips' package.\n", + "usage_str": "To use, type `geoips test script -p <--integration> `.\n", + "output_info": [ + "Output of the Script Being Ran" + ] + }, + "test_unit-test": { + "help_str": "Run tests unit-test[s] implemented in a certain GeoIPS package. This command\nexpects that the unit tests implemented are pytest-based. Defaults to the 'geoips'\npackage.\n", + "usage_str": "To use, type\n`geoips test unit-test -p <-n> `\n", + "output_info": [ + "Output of the Unit Test Being Ran" + ] + }, + "validate": { + "help_str": "Validate a GeoIPS Plugin found at . While this is done under the hood via\nGeoIPS, this is an easy way to test whether or not the plugin you are developing\nis valid.\n", + "usage_str": "To use, type `geoips validate `. Do it man. NOW.\n", + "output_info": [ + "A message telling you if the plugin is valid or not and why." + ] + } + } +} \ No newline at end of file diff --git a/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.yaml b/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.yaml new file mode 100644 index 000000000..aa28d3b47 --- /dev/null +++ b/tests/unit_tests/commandline/cmd_instructions/yaml_newer/cmd_instructions.yaml @@ -0,0 +1,412 @@ +name: 2024-05-20_19:43:53_UTC +instructions: + config: + help_str: 'Various configuration-based commands for setting up your geoips environment. + + Currently supports `geoips config install ` command. + + ' + output_info: + - Output related to config process that was run. + usage_str: 'To use, type `geoips config ...` + + ' + config_install: + help_str: 'Install the appropriate test dataset and/or package based on the arguments + + provided. To see a list of available test datasets for install, run + + `geoips list test-datasets`. + + ' + output_info: + - Not Applicable + usage_str: 'To use, type `geoips config install `. + + ' + get: + help_str: 'Retrieve an appropriate GeoIPS artifact. Currently supported `get` + calls are: + + ["family", "interface", "package", "plugin"] + + ' + output_info: + - Information related to the artifact that was retrieved. + usage_str: 'To use, type `geoips get ...`. + + ' + get_family: + help_str: 'Retrieve the appropriate GeoIPS Family alongside descriptive information + of + + that family. See output_info for each datum provided when this command is + + called. For a listing of available GeoIPS families, run: + + `geoips list interfaces`. + + ' + output_info: + - Docstring + - Family Name + - Family Path + - Interface Name + - Interface Type + - Required Args / Schema + usage_str: 'To use, type `geoips get family `, where + + is a valid GeoIPS Interface and is a supported + + family member of that interface. + + ' + get_interface: + help_str: 'Retrieve the appropriate GeoIPS Interface alongside descriptive information + of + + that interface. See output_info for each datum provided when this command is + + called. For a listing of available GeoIPS interfaces, run: + + `geoips list interfaces`. + + ' + output_info: + - Absolute Path + - Docstring + - Documentation Link (If applicable) + - Interface Name + - Interface Type + - Supported Families + usage_str: 'To use, type `geoips get interface `, where + is a + + valid GeoIPS Interface. + + ' + get_package: + help_str: 'Retrieve the appropriate GeoIPS Package alongside descriptive information + of + + that package. See output_info for each datum provided when this command is + + called. For a listing of available GeoIPS packages, run: + + `geoips list packages`. + + ' + output_info: + - GeoIPS Package + - Docstring + - Package Path + - Documentation Link + usage_str: 'To use, type `geoips get package `, where + is a + + valid GeoIPS Package. + + ' + get_plugin: + help_str: 'Retrieve the appropriate GeoIPS Plugin alongside descriptive information + of + + that plugin. See output_info for each datum provided when this command is + + called. For a listing of available GeoIPS Plugins, run: + + `geoips list plugins`. + + ' + output_info: + - Docstring + - Documentation Link (If applicable) + - Family + - Interface Name + - Package + - Relative Path + - Plugin Specific Info (signature / source_names / available overrides) + usage_str: 'To use, type `geoips get plugin `, where + + is a valid GeoIPS Interface and is a valid plugin + + identifier that has been implemented in any installed GeoIPS package. + + ' + list: + help_str: 'List off available GeoIPS artifacts provided further commands. Current + artifacts + + available for listing are: ["interface", "interfaces", "packages", "plugins", + + "scripts", "test-datasets", "unit-tests"]. + + ' + output_info: + - Additional info related to the artifact[s] being listed. + usage_str: 'To use, type `geoips list `. + + ' + list_interface: + help_str: 'List off plugins found under a certain interface. This is useful for + seeing what + + plugins have been implemented through all, or a certain, GeoIPS Package. This + + information can then be used with + + `geoips get-plugin ` to get more information about + a + + certain plugin. + + ' + output_info: + - Family + - Interface Name + - Interface Type + - Package + - Plugin Name + - Relative Path + usage_str: 'To use, type `geoips list interface `. Optionally + includes `-p` + + flag for seeing what plugins are in a certain package (-p). For example, to + see + + what plugins under a certain interface have been created in a single package, + + run `geoips list interface -p `, or you can remove + the + + `-p` flag to see what''s been created in every package. + + ' + list_interfaces: + help_str: 'List off every GeoIPS Interface. This is useful for seeing what is + + available throughout GeoIPS, but can also be extended to see what interfaces + have + + been implemented in other geoips package, or just a certain package. This + + information can then be used with `geoips get-interface ` to + get + + more information about a certain interface. + + ' + output_info: + - Absolute Path + - Docstring + - Documentation Link (if applicable) + - Interface Name + - Interface Type + - Package + - Supported Families + usage_str: 'To use, type `geoips list interfaces`. Optionally includes `-i` and + `-p` flags for + + seeing what has been implemented (-i) in a certain package (-p), or all packages. + + For example, to see what interfaces have been implemented in a certain package, + + run `geoips list interfaces -i -p `, or you can remove the `-p` flag + to + + see what''s been implemented in every package. + + ' + list_packages: + help_str: 'List off packages found under the GeoIPS Namespace, alongside their + path and + + top-level docstring. This command will list every GeoIPS Package found, provided + + it has already been installed. + + ' + output_info: + - Docstring + - Package + - Relative Path + usage_str: 'To use, type `geoips list packages`. + + ' + list_plugins: + help_str: 'List off plugins found under all, or a specified GeoIPS package. This + information + + can then be used with `geoips get-plugin ` to + get + + more information about a certain plugin. + + ' + output_info: + - Family + - Interface Name + - Interface Type + - Package + - Plugin Name + - Relative Path + usage_str: 'To use, type `geoips list plugins`. Optionally includes `-p` flag + for + + seeing what plugins are in a certain package (-p). For example, to see what + + plugins have been created in a single package, run + + `geoips list plugins -p `, or you can remove the `-p` flag to see + what''s + + been created in every package. + + ' + list_scripts: + help_str: 'List off scripts found under all, or a specified GeoIPS package. This + information + + can then be used with `geoips run ` to run a certain + + process workflow, defined in a bash script. + + ' + output_info: + - Package + - Script Name + usage_str: 'To use, type `geoips list scripts`. Optionally includes `-p` flag + for + + seeing what scripts are in a certain package (-p). For example, to see what + + scripts have been created in a single package, run + + `geoips list scripts -p `, or you can remove the `-p` flag to see + what''s + + been created in every package. + + ' + list_test-datasets: + help_str: 'List off test-datasets used for testing GeoIPS and other GeoIPS Packages. + + This data is used for running tests via `geoips run `. + + ' + output_info: + - Data Host + - Dataset Name + usage_str: 'To use, type `geoips list test-datasets`. + + ' + list_unit-tests: + help_str: 'List off unit-tests used in a certain GeoIPS Package. + + This data can be used to run unit tests via `geoips test unit-test ...`. + + ' + output_info: + - GeoIPS Package + - Unit Test Directory + - Unit Test Name + usage_str: 'To use, type `geoips list unit-tests -p `. + + ' + run: + help_str: 'Run a geoips process workflow (procflow), given a GeoIPS package and + a script + + name. This will do the exact same thing as being in the directory of a certain + + script, and running it via `./`. To see what scripts are available, + + you can run `geoips list-scripts <-p>`, where `-p` can be used alongside a GeoIPS + + Package name if you want scripts from a single package. + + ' + output_info: + - Log Output of the Process Workflow (Procflow) + usage_str: 'To use, type `geoips run -p `. + + ' + test: + help_str: 'Run tests implemented in GeoIPS or a separate GeoIPS Package. This + could be + + running a set of linters on a specific package, running a certain integration + or + + normal test script, or running unit tests. Current options are ["linting", + + "script", "unit-test"]. + + ' + output_info: + - Output of the Test Being Ran + usage_str: 'To use, type `geoips test `. + + ' + test_linting: + help_str: 'Run all GeoIPS linters on a specific GeoIPS Package. This will test + whether or not + + the code you wrote adheres to the specified coding conventions set up by GeoIPS. + + Defaults to the ''geoips'' package. + + ' + output_info: + - Output of the Linters + usage_str: 'To use, type `geoips test linting -p `. + + ' + test_script: + help_str: 'Runs a specific test script (integration-based or normal), found within + a certain + + GeoIPS package. If this is specified as an integration test, this will only + work + + using the ''geoips'' package. Use ''--integration'' to specify that the script + + is integration-based. Defaults to the ''geoips'' package. + + ' + output_info: + - Output of the Script Being Ran + usage_str: 'To use, type `geoips test script -p <--integration> + `. + + ' + test_unit-test: + help_str: 'Run tests unit-test[s] implemented in a certain GeoIPS package. This + command + + expects that the unit tests implemented are pytest-based. Defaults to the ''geoips'' + + package. + + ' + output_info: + - Output of the Unit Test Being Ran + usage_str: 'To use, type + + `geoips test unit-test -p <-n> ` + + ' + validate: + help_str: 'Validate a GeoIPS Plugin found at . While this is done under + the hood via + + GeoIPS, this is an easy way to test whether or not the plugin you are developing + + is valid. + + ' + output_info: + - A message telling you if the plugin is valid or not and why. + usage_str: 'To use, type `geoips validate `. Do it man. NOW. + + ' diff --git a/tests/unit_tests/commandline/test_expose.py b/tests/unit_tests/commandline/test_expose.py index 52b16f3e6..8695eb3d5 100644 --- a/tests/unit_tests/commandline/test_expose.py +++ b/tests/unit_tests/commandline/test_expose.py @@ -1,3 +1,6 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + """Unit test asserting functionality for exposing plugin-package commands.""" from importlib import metadata diff --git a/tests/unit_tests/commandline/test_geoips_test_sector.py b/tests/unit_tests/commandline/test_geoips_test_sector.py index 66f3d8f50..6625e10c4 100644 --- a/tests/unit_tests/commandline/test_geoips_test_sector.py +++ b/tests/unit_tests/commandline/test_geoips_test_sector.py @@ -1,14 +1,5 @@ -# # # Distribution Statement A. Approved for public release. Distribution is unlimited. -# # # -# # # Author: -# # # Naval Research Laboratory, Marine Meteorology Division -# # # -# # # This program is free software: you can redistribute it and/or modify it under -# # # the terms of the NRLMMD License included with this program. This program is -# # # distributed WITHOUT ANY WARRANTY; without even the implied warranty of -# # # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the included license -# # # for more details. If you did not receive the license, for more information see: -# # # https://github.com/U-S-NRL-Marine-Meteorology-Division/ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. """Unit test for GeoIPS CLI `test sector` command. diff --git a/tests/unit_tests/commandline/test_geoips_tree.py b/tests/unit_tests/commandline/test_geoips_tree.py index a76be8ded..6a2acde7c 100644 --- a/tests/unit_tests/commandline/test_geoips_tree.py +++ b/tests/unit_tests/commandline/test_geoips_tree.py @@ -1,3 +1,6 @@ +# # # This source code is protected under the license referenced at +# # # https://github.com/NRLMMD-GEOIPS. + """Unit test for GeoIPS CLI `tree` command. See geoips/commandline/ancillary_info/cmd_instructions.yaml for more information. diff --git a/tests/utils/check_code.sh b/tests/utils/check_code.sh index 0d738a32b..9fb5b0798 100755 --- a/tests/utils/check_code.sh +++ b/tests/utils/check_code.sh @@ -176,6 +176,7 @@ if [[ "$test" == "pytest_short" || "$test" == "all" ]]; then fi echo "TEST COMPLETE pytest_short" fi +# Do not include in "test" keyword - this takes a very long time to run if [[ "$test" == "pytest_long" || "$test" == "all_test_data" ]]; then echo "" echo "CALLING TEST:"