Merge branch 'NOAA-EMC:develop' into develop

.github/pull_request_template.md

@@ -33,6 +33,17 @@
 # Change characteristics
 - Is this a breaking change (a change in existing functionality)? YES/NO
 - Does this change require a documentation update? YES/NO
+- Does this change require an update to any of the following submodules? YES/NO (If YES, please add a link to any PRs that are pending.)
+  - [ ] EMC verif-global
+  - [ ] GDAS
+  - [ ] GFS-utils
+  - [ ] GSI
+  - [ ] GSI-monitor
+  - [ ] GSI-utils
+  - [ ] UFS-utils
+  - [ ] UFS-weather-model
+  - [ ] wxflow
+
 
 # How has this been tested?
 <!-- Please list any test you conducted, including the machine.

.github/workflows/ci_unit_tests.yaml

@@ -0,0 +1,64 @@
+name: CI Unit Tests
+on: [pull_request, push, workflow_dispatch]
+
+jobs:
+
+  ci_pytest:
+      runs-on: ubuntu-latest
+      name: Run unit tests on CI system      
+      permissions:
+        checks: write
+
+      steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11.8
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y perl libxml-libxml-perl libxml-libxslt-perl libdatetime-perl
+          python -m pip install --upgrade pip
+          pip install pytest
+          pip install wxflow
+          pip install wget
+
+      - name: Cache Rocoto Install
+        uses: actions/cache@v4
+        with:
+          path: ~/rocoto
+          key: ${{ runner.os }}-rocoto-${{ hashFiles('**/ci-unit_tests.yaml') }}
+
+      - name: Install Rocoto
+        run: |
+          if [ ! -d "$HOME/rocoto/bin" ]; then
+            git clone https://github.com/christopherwharrop/rocoto.git $HOME/rocoto
+            cd $HOME/rocoto
+            ./INSTALL
+          fi
+          echo "$HOME/rocoto/bin" >> $GITHUB_PATH
+
+      - name: Run tests
+        shell: bash
+        run: |
+          sudo mkdir -p /scratch1/NCEPDEV
+          cd $GITHUB_WORKSPACE/sorc
+          git submodule update --init
+          ./link_workflow.sh
+          cd $GITHUB_WORKSPACE/ci/scripts/tests
+          ln -s ../wxflow
+
+          pytest -v --junitxml $GITHUB_WORKSPACE/ci/scripts/tests/test-results.xml
+
+
+      - name: Publish Test Results
+        if: always()
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          files: ci/scripts/tests/test-results.xml
+          job_summary: true
+          comment_mode: off

ci/cases/gfsv17/ocnanal.yaml

@@ -14,17 +14,14 @@ base:
   DO_VERFRAD: "YES"
   DO_VRFY_OCEANDA: "NO"
   FHMAX_GFS: 240
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
 
 ocnanal:
-  SOCA_INPUT_FIX_DIR: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/1440x1080x75/soca
-  CASE_ANL: 'C24'
+  SOCA_INPUT_FIX_DIR: {{ FIXgfs }}/gdas/soca/1440x1080x75/soca
   SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
   SOCA_NINNER: 100
-  SABER_BLOCKS_YAML: ''
-  NICAS_RESOL: 1
-  NICAS_GRID_SIZE: 15000
 
 prepoceanobs:
-  SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml 
+  SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
   OBSPREP_YAML: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml
   DMPDIR: /scratch1/NCEPDEV/da/common/

ci/cases/pr/C96_atmaerosnowDA.yaml

@@ -19,4 +19,3 @@ arguments:
 skip_ci_on_hosts:
   - orion
   - hercules
-  - wcoss2

ci/cases/yamls/gefs_ci_defaults.yaml

@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml
 base:
-  HPC_ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}

ci/scripts/check_ci.sh

@@ -175,7 +175,7 @@ for pr in ${pr_list}; do
       DATE=$(date +'%D %r')
       echo "Experiment ${pslot} **SUCCESS** on ${MACHINE_ID^} at ${DATE}" >> "${output_ci_single}"
       echo "Experiment ${pslot} *** SUCCESS *** at ${DATE}" >> "${output_ci}"
-      "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
+      # "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
     fi
   done
 done

ci/scripts/run-check_ci.sh

@@ -101,9 +101,7 @@ while true; do
 
   if [[ "${ROCOTO_STATE}" == "DONE" ]]; then
     {
-      echo "Experiment ${pslot} Completed ${CYCLES_DONE} Cycles at $(date)" || true
-      echo "with ${SUCCEEDED} successfully completed jobs" || true
-      echo "Experiment ${pslot} Completed: *SUCCESS*"
+      echo "Experiment ${pslot} Completed ${CYCLES_DONE} Cycles: *SUCCESS* at $(date)" || true
     } | tee -a "${run_check_logfile}"
     rc=0
     break

ci/scripts/tests/test_create_experiment.py

@@ -0,0 +1,29 @@
+from wxflow import Executable
+from shutil import rmtree
+import os
+import copy
+
+_here = os.path.dirname(__file__)
+HOMEgfs = os.sep.join(_here.split(os.sep)[:-3])
+RUNDIR = os.path.join(_here, 'testdata/RUNDIR')
+
+
+def test_create_experiment():
+
+    create_experiment_script = Executable(f'{HOMEgfs}/workflow/create_experiment.py')
+    yaml_dir = yaml_dir = os.path.join(HOMEgfs, 'ci/cases/pr')
+    env = os.environ.copy()
+    env['RUNTESTS'] = RUNDIR
+
+    for case in os.listdir(yaml_dir):
+        if case.endswith('.yaml'):
+            with open(os.path.join(yaml_dir, case), 'r') as file:
+                file_contents = file.read()
+                if 'ICSDIR_ROOT' not in file_contents:
+                    create_experiment = copy.deepcopy(create_experiment_script)
+                    create_experiment.add_default_arg(['-y', f'../../cases/pr/{case}', '--overwrite'])
+                    env['pslot'] = os.path.splitext(case)[0]
+                    create_experiment(env=env)
+                    assert (create_experiment.returncode == 0)
+
+    rmtree(RUNDIR)

ci/scripts/tests/test_rocotostat.py

@@ -0,0 +1,89 @@
+import sys
+import os
+from shutil import rmtree
+import wget
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.join(os.path.dirname(script_dir), 'utils'))
+
+from rocotostat import rocoto_statcount, rocotostat_summary, is_done, is_stalled, CommandNotFoundError
+from wxflow import which
+
+test_data_url = 'https://noaa-nws-global-pds.s3.amazonaws.com/data/CI/'
+
+testdata_path = 'testdata/rocotostat'
+testdata_full_path = os.path.join(script_dir, testdata_path)
+
+
+if not os.path.isfile(os.path.join(testdata_full_path, 'database.db')):
+    os.makedirs(testdata_full_path, exist_ok=True)
+    workflow_url = test_data_url + str(testdata_path) + '/workflow.xml'
+    workflow_destination = os.path.join(testdata_full_path, 'workflow.xml')
+    wget.download(workflow_url, workflow_destination)
+
+    database_url = test_data_url + str(testdata_path) + '/database.db'
+    database_destination = os.path.join(testdata_full_path, 'database.db')
+    wget.download(database_url, database_destination)
+
+rocotostat_cmd = which('rocotostat')
+if not rocotostat_cmd:
+    raise CommandNotFoundError("rocotostat not found in PATH")
+
+rocotostat_cmd.add_default_arg(['-w', os.path.join(testdata_path, 'workflow.xml'), '-d', os.path.join(testdata_path, 'database.db')])
+
+
+def test_rocoto_statcount():
+
+    result = rocoto_statcount(rocotostat_cmd)
+
+    assert result['SUCCEEDED'] == 20
+    assert result['FAIL'] == 0
+    assert result['DEAD'] == 0
+    assert result['RUNNING'] == 0
+    assert result['SUBMITTING'] == 0
+    assert result['QUEUED'] == 0
+
+
+def test_rocoto_summary():
+
+    result = rocotostat_summary(rocotostat_cmd)
+
+    assert result['CYCLES_TOTAL'] == 1
+    assert result['CYCLES_DONE'] == 1
+
+
+def test_rocoto_done():
+
+    result = rocotostat_summary(rocotostat_cmd)
+
+    assert is_done(result)
+
+    rmtree(testdata_full_path)
+
+
+def test_rocoto_stalled():
+    testdata_path = 'testdata/rocotostat_stalled'
+    testdata_full_path = os.path.join(script_dir, testdata_path)
+    xml = os.path.join(testdata_full_path, 'stalled.xml')
+    db = os.path.join(testdata_full_path, 'stalled.db')
+
+    if not os.path.isfile(os.path.join(testdata_full_path, 'stalled.db')):
+        os.makedirs(testdata_full_path, exist_ok=True)
+        workflow_url = test_data_url + str(testdata_path) + '/stalled.xml'
+        database_url = test_data_url + str(testdata_path) + '/stalled.db'
+
+        workflow_destination = os.path.join(testdata_full_path, 'stalled.xml')
+        wget.download(workflow_url, workflow_destination)
+
+        database_destination = os.path.join(testdata_full_path, 'stalled.db')
+        wget.download(database_url, database_destination)
+
+    rocotostat_cmd = which('rocotostat')
+    rocotostat_cmd.add_default_arg(['-w', xml, '-d', db])
+
+    result = rocoto_statcount(rocotostat_cmd)
+
+    assert result['SUCCEEDED'] == 11
+    assert is_stalled(result)
+
+    rmtree(testdata_full_path)

ci/scripts/tests/test_setup.py

@@ -0,0 +1,89 @@
+from wxflow import Executable, Configuration, ProcessError
+from shutil import rmtree
+import pytest
+import os
+
+_here = os.path.dirname(__file__)
+HOMEgfs = os.sep.join(_here.split(os.sep)[:-3])
+RUNDIR = os.path.join(_here, 'testdata/RUNDIR')
+pslot = "C48_ATM"
+account = "fv3-cpu"
+foobar = "foobar"
+
+
+def test_setup_expt():
+
+    arguments = [
+        "gfs", "forecast-only",
+        "--pslot", pslot, "--app", "ATM", "--resdetatmos", "48",
+        "--comroot", f"{RUNDIR}", "--expdir", f"{RUNDIR}",
+        "--idate", "2021032312", "--edate", "2021032312", "--overwrite"
+    ]
+    setup_expt_script = Executable(os.path.join(HOMEgfs, "workflow", "setup_expt.py"))
+    setup_expt_script.add_default_arg(arguments)
+    setup_expt_script()
+    assert (setup_expt_script.returncode == 0)
+
+
+def test_setup_xml():
+
+    setup_xml_script = Executable(os.path.join(HOMEgfs, "workflow/setup_xml.py"))
+    setup_xml_script.add_default_arg(f"{RUNDIR}/{pslot}")
+    setup_xml_script()
+    assert (setup_xml_script.returncode == 0)
+
+    cfg = Configuration(f"{RUNDIR}/{pslot}")
+    base = cfg.parse_config('config.base')
+    assert base.ACCOUNT == account
+
+    assert "UNKNOWN" not in base.values()
+
+    with open(f"{RUNDIR}/{pslot}/{pslot}.xml", 'r') as file:
+        contents = file.read()
+    assert contents.count(account) > 5
+
+    rmtree(RUNDIR)
+
+
+def test_setup_xml_fail_config_env_cornercase():
+
+    script_content = ('''#!/usr/bin/env bash
+export HOMEgfs=foobar
+../../../workflow/setup_xml.py "${1}"\n
+''')
+
+    with open('run_setup_xml.sh', 'w') as file:
+        file.write(script_content)
+    os.chmod('run_setup_xml.sh', 0o755)
+
+    try:
+        setup_xml_script = Executable(os.path.join(HOMEgfs, "ci", "scripts", "tests", "run_setup_xml.sh"))
+        setup_xml_script.add_default_arg(f"{RUNDIR}/{pslot}")
+        setup_xml_script()
+        assert (setup_xml_script.returncode == 0)
+
+        cfg = Configuration(f"{RUNDIR}/{pslot}")
+        base = cfg.parse_config('config.base')
+        assert base.ACCOUNT == account
+
+        assert foobar not in base.values()
+        assert "UNKNOWN" not in base.values()
+
+        with open(f"{RUNDIR}/{pslot}/{pslot}.xml", 'r') as file:
+            contents = file.read()
+        assert contents.count(account) > 5
+
+    except ProcessError as e:
+        # We expect this fail becuse ACCOUNT=fv3-cpu in config.base and environment
+        pass
+    except Exception as e:
+        # If an exception occurs, pass the test with a custom message
+        pytest.fail(f"Expected exception occurred: {e}")
+
+    finally:
+        # Cleanup code to ensure it runs regardless of test outcome
+        os.remove('run_setup_xml.sh')
+        try:
+            rmtree(RUNDIR)
+        except FileNotFoundError:
+            pass