Integration of new PFB IO functionality and xarray compatibility (par…

…flow#365)

Add simple and fast pure-python based readers and writers of PFB files, done by myself and Bill Hasling (@wh3248). This eliminates the need for the external ParflowIO dependency. Implemented a new backend for the xarray package that let's you open both .pfb files as well as .pfmetadata files directly into xarray datastructures. These are very useful for data wrangling and scientific analysis

Basic usage of the new functionality:

import parflow as pf

# Read a pfb file as numpy array:
x = pf.read_pfb('/path/to/file.pfb')

# Read a pfb file as an xarray dataset:
ds = xr.open_dataset('/path/to/file.pfb', name='example')

# Write a pfb file with distfile:
pf.write_pfb('/path/to/new_file.pfb', x, 
             p=p, q=q, r=r, dist=True)

.gitignore

@@ -109,6 +109,55 @@
 ### Python ###
 __pycache__
 *.pyc
+*.py[cod]
+*$py.class
 
 # PyCharm
 .idea
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+# Sphinx documentation
+docs/_build/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version

pftools/python/CMakeLists.txt

@@ -55,7 +55,6 @@ install(
     "${CMAKE_CURRENT_SOURCE_DIR}/requirements.txt"
     "${CMAKE_CURRENT_SOURCE_DIR}/requirements_all.txt"
     "${CMAKE_CURRENT_SOURCE_DIR}/requirements_pfsol.txt"
-    "${CMAKE_CURRENT_SOURCE_DIR}/requirements_pfb.txt"
     "${CMAKE_CURRENT_SOURCE_DIR}/README.md"
     "${CMAKE_CURRENT_SOURCE_DIR}/setup.py"
   DESTINATION

pftools/python/README.md

@@ -1,7 +1,7 @@
 # pftools
 
-This is a package to run ParFlow via a Python interface. This package allows 
-the user to build a script in Python that builds the database (.pfidb file) 
+This is a package to run ParFlow via a Python interface. This package allows
+the user to build a script in Python that builds the database (.pfidb file)
 which ParFlow reads as input.
 
 ## How to use this package
@@ -45,64 +45,64 @@ ParFlow run object:
     messages. This does not require ParFlow.
     - `write(file_name=None, file_format='pfidb')`: This will write your key/value
     pairs to a file with your choice of format (default is the ParFlow database `pfidb`
-    format). Other acceptable formats passed as the `file_format` argument include 
-    `yml`, `yaml`, and `json`. This method does not require ParFlow. 
+    format). Other acceptable formats passed as the `file_format` argument include
+    `yml`, `yaml`, and `json`. This method does not require ParFlow.
     - `clone(name)`: This will generate a clone object of your run with the given `name`.
     See `parflow/test/python/new_features/serial_runs/serial_runs.py` for an example of
-    how to use this. 
-    - `run(working_directory=None, skip_validation=False)`: This will execute the 
-    `write()` method. If `skip_validation` is set to `False`, it will also execute the 
+    how to use this.
+    - `run(working_directory=None, skip_validation=False)`: This will execute the
+    `write()` method. If `skip_validation` is set to `False`, it will also execute the
     `validate()` method. The `working_directory` can be defined as an argument if you
-    would like to change the directory where the output files will be written, but it 
-    defaults to the directory of the Python script. Finally, `run()` will execute 
-    ParFlow. This will print the data for your environment (ParFlow directory, 
+    would like to change the directory where the output files will be written, but it
+    defaults to the directory of the Python script. Finally, `run()` will execute
+    ParFlow. This will print the data for your environment (ParFlow directory,
     ParFlow version, working directory, and the generated ParFlow database file).
-    If ParFlow runs successfully, you will get a message `ParFlow ran successfully`. 
-    Otherwise, you will get a message `ParFlow run failed.` followed by a print of the 
-    contents of the `runname.out.txt` file. 
+    If ParFlow runs successfully, you will get a message `ParFlow ran successfully`.
+    Otherwise, you will get a message `ParFlow run failed.` followed by a print of the
+    contents of the `runname.out.txt` file.
 
 
 6. Once you have completed your input script, save and run it via the Python terminal
 or command line:
 
         python3 runname.py
-        
+
     You can append one or more of the following arguments to the run:
-    
-    - `--parflow-directory [None]`: overrides environment variable for 
+
+    - `--parflow-directory [None]`: overrides environment variable for
     `$PARFLOW_DIR`.
     - `--parflow-version [None]`: overrides the sourced version of ParFlow used to validate
-    keys. 
+    keys.
     - `--working-directory [None]`: overrides the working directory for the ParFlow run.
     This is identical to specifying `working_directory` in the `run()` method.
-    - `--skip-validation [False]`: skips the `validate()` method if set to `True`. This is 
+    - `--skip-validation [False]`: skips the `validate()` method if set to `True`. This is
     identical to specifying `skip_validation` in the `run()` method.
     - `--show-line-error [False]`: shows the line where an error occurs when set to `True`.
     - `--exit-on-error [False]`: causes the run to exit whenever it encounters an error when
     set to `True`.
-    - `--write-yaml [False]`: writes the key/value pairs to a yaml file when set to `True`. 
+    - `--write-yaml [False]`: writes the key/value pairs to a yaml file when set to `True`.
     This is identical to calling the method `runname.write(file_format='yaml)`.
     - `-p [0]`: overrides the value for `Process.Topology.P` (must be an integer).
     - `-q [0]`: overrides the value for `Process.Topology.Q` (must be an integer).
     - `-r [0]`: overrides the value for `Process.Topology.R` (must be an integer).
-    
+
 ## How to update this package (developers only)
 
-This assumes that you are using CMake with the pftools package as it is 
+This assumes that you are using CMake with the pftools package as it is
 contained within the main ParFlow repo (see https://github.com/parflow/parflow)
 
 1. Update the version number in setup.py
 
-2. Run the following command to create and test a source archive and a wheel 
-   distribution of the package: 
+2. Run the following command to create and test a source archive and a wheel
+   distribution of the package:
 
         make PythonCreatePackage
 
 3. If the distributions pass, run the following command to publish the
-   distributions: 
+   distributions:
 
         make PythonPublishPackage
-        
+
 4. Check PyPI to make sure your package update was published correctly. Thanks
    for contributing!
 

pftools/python/parflow/__init__.py

@@ -5,5 +5,12 @@
 
 """
 from .tools import Run
+from .tools import ParflowBinaryReader, read_pfb, read_pfb_sequence, write_pfb
 
-__all__ = ['Run']
+__all__ = [
+    'Run',
+    'ParflowBinaryReader',
+    'read_pfb',
+    'write_pfb',
+    'read_pfb_sequence',
+]

pftools/python/parflow/tools/__init__.py

@@ -1,9 +1,16 @@
 # -*- coding: utf-8 -*-
 """parflow.tools module
 
-Export Run() object
+Export Run() object and IO functions
 
 """
 from .core import Run
+from .io import ParflowBinaryReader, read_pfb, read_pfb_sequence, write_pfb
 
-__all__ = ['Run']
+__all__ = [
+    'Run',
+    'ParflowBinaryReader',
+    'read_pfb',
+    'write_pfb',
+    'read_pfb_sequence',
+]

pftools/python/parflow/tools/builders.py

@@ -5,6 +5,8 @@
 import sys
 import tempfile
 import yaml
+import subprocess
+from subprocess import Popen, PIPE
 
 import numpy as np
 
@@ -13,7 +15,7 @@
 from .fs import exists
 
 from parflow.tools.database.core import PFDBObj
-from parflow.tools.io import read_clm, write_array, write_patch_matrix_as_asc, write_array_pfb
+from parflow.tools.io import read_clm, write_pfb, write_patch_matrix_as_asc
 from parflow.tools.fs import get_absolute_path
 from parflow.tools.helper import remove_prefix, with_absolute_path
 
@@ -214,7 +216,9 @@ def write(self, name, xllcorner=0, yllcorner=0, cellsize=0, vtk=False, extra=Non
 
         else:
             temp_pfb_file = tempfile.NamedTemporaryFile(suffix='.pfb')
-            write_array_pfb(temp_pfb_file.name, self.mask_array)
+            if self.mask_array.dtype != np.float64:
+                self.mask_array = self.mask_array.astype(np.float64)
+            write_pfb(temp_pfb_file.name, self.mask_array)
             args = [
                 f'--mask {temp_pfb_file.name}',
                 f'--side-patch-label {self.side_id}',
@@ -233,8 +237,13 @@ def write(self, name, xllcorner=0, yllcorner=0, cellsize=0, vtk=False, extra=Non
         exe_path = get_absolute_path('$PARFLOW_DIR/bin/pfmask-to-pfsol')
         args = args + extra
         cmd_line = f'{exe_path} ' + ' '.join(args)
-        print(f'$ {cmd_line}')
-        os.system(cmd_line)
+        process = Popen(cmd_line.split(), stdout=PIPE, stderr=PIPE)
+        stdout, stderr = process.communicate()
+        print('Standard output:')
+        print(stdout)
+        print('')
+        print('Standard error:')
+        print(stderr)
 
         print('=== pfmask-to-pfsol ===: END')
 
@@ -1376,7 +1385,7 @@ def map(self, vegm_data):
                 item.Type = 'Constant'
                 item.Value = array[0, 0].item()
             else:
-                write_array(get_absolute_path(file_name), vegm_data[:, :, i])
+                write_pfb(get_absolute_path(file_name), vegm_data[:, :, i])
                 item.Type = 'PFBFile'
                 item.FileName = file_name
 

pftools/python/parflow/tools/core.py

@@ -15,7 +15,14 @@
 
 from . import settings
 from .fs import get_absolute_path
-from .io import write_dict, DataAccessor
+from .io import (
+    DataAccessor,
+    ParflowBinaryReader,
+    write_dict,
+    read_pfb,
+    write_pfb,
+    write_dist
+)
 from .terminal import Symbols as TermSymbol
 
 from .database.generated import BaseRun
@@ -489,6 +496,14 @@ def dist(self, pfb_file, **kwargs):
         q = kwargs.get('Q', self.Process.Topology.Q)
         r = kwargs.get('R', self.Process.Topology.R)
 
-        from parflowio.pyParflowio import PFData
-        pfb_data = PFData(pfb_file_full_path)
-        pfb_data.distFile(p, q, r, pfb_file_full_path)
+        with ParflowBinaryReader(pfb_file_full_path) as pfb:
+            array = pfb.read_all_subgrids()
+            header = pfb.header
+
+        dx, dy, dz = header['dx'], header['dy'], header['dz']
+        write_pfb(pfb_file_full_path, array,
+                   p=p, q=q, r=r, dx=dx, dy=dy, dz=dz,
+                   dist=True)
+
+
+

pftools/python/parflow/tools/export.py

@@ -13,7 +13,7 @@
 
 from .fs import get_absolute_path
 
-from parflow.tools.io import read_array
+from parflow.tools.io import read_pfb
 from parflow.tools.database.core import PFDBObj
 from parflow.tools.database.generated import LandCoverParamItem, CLM_KEY_DICT
 
@@ -245,7 +245,7 @@ def _process_vegm(self, token, x, y, axis=None):
                 else:
                     raise Exception(f'Axis specification error: {axis}')
         elif vegm_root_key.Type == 'PFBFile':
-            array = read_array(get_absolute_path(vegm_root_key.FileName))
+            array = read_pfb(get_absolute_path(vegm_root_key.FileName))
             while array.ndim > 2:
                 # PFB files return 3D arrays, but the data is actually 2D
                 array = array[0]