Development

environment.yml

@@ -17,4 +17,5 @@ dependencies:
   - openssl
   - fastapi
   - fire
+  - hydra-core
 prefix: /home/gjb/mambaforge/envs/python_for_systems_programming

hands-on/shell_interaction.ipynb

@@ -51,20 +51,13 @@
     "sh.ls('-l')"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The output can be used by assigning the command to a variable, and using the result's `stdout` attribute.  Note that the latter is a sequence of bytes, so it has to be decoded into a UTF-8 string for further processing."
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "cmd = sh.ls('-l', '-a', _encoding='UTF-8')"
+    "result = sh.ls('-l', '-a')"
    ]
   },
   {
@@ -73,7 +66,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "lines = cmd.stdout.decode(encoding='utf8').split('\\n')"
+    "lines = result.split('\\n')"
    ]
   },
   {
@@ -109,7 +102,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sh.ls()"
+    "for file in (line.strip() for line in sh.ls().split()):\n",
+    "    print(file)"
    ]
   },
   {
@@ -179,7 +173,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sh.cat('tmp/date_file.txt')"
+    "print(sh.cat('tmp/date_file.txt'))"
    ]
   },
   {
@@ -228,14 +222,14 @@
    },
    "outputs": [],
    "source": [
-    "sh.grep(sh.ls('-l'), r'\\.ipynb$')"
+    "print(sh.grep('-e', r'\\.ipynb$', _in=sh.ls('-l')))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Pipe the output of `cut` into `sort`."
+    "Pipe the output of `cut` into `sort`.  Also use the `_iter` argument to create a generator over standard output."
    ]
   },
   {
@@ -246,7 +240,8 @@
    },
    "outputs": [],
    "source": [
-    "sh.sort(sh.cut('-d', ' ', '-f', '5', 'tmp/date_file.txt'), '-r')"
+    "for line in sh.sort('-r', _in=sh.cut('-d', ' ', '-f', '5', 'tmp/date_file.txt'), _iter=True):\n",
+    "    print(line.strip())"
    ]
   },
   {
@@ -315,10 +310,12 @@
    },
    "outputs": [],
    "source": [
+    "process = sh.sleep(10, _bg=True, _bg_exc=False, _timeout=3)\n",
     "try:\n",
-    "    process = sh.sleep(10, _bg=True, _timeout=3)\n",
-    "except TimeoutError as error:\n",
-    "    print(error)"
+    "    process.wait()\n",
+    "except sh.TimeoutException as error:\n",
+    "    print('process timed out')\n",
+    "    print(error.exit_code)"
    ]
   },
   {
@@ -578,7 +575,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "process = subprocess.run('ls *.py', stdout=subprocess.PIPE, encoding='utf8', shell=True)"
+    "process = subprocess.run('ls *.ipynb', stdout=subprocess.PIPE, encoding='utf8', shell=True)"
    ]
   },
   {
@@ -653,7 +650,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [],
    "source": [
     "process.returncode"
@@ -676,7 +675,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,

python_for_systems_programming_linux64_conda_specs.txt

@@ -4,7 +4,7 @@
 @EXPLICIT
 https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2
 https://repo.anaconda.com/pkgs/main/linux-64/blas-1.0-openblas.conda
-https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.6.2-hbcca054_0.conda
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2
@@ -53,7 +53,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/lz4-c-1.9.4-h6a678d5_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/mpg123-1.32.6-h59595ed_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.5-hd590300_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.6-h4ab18f5_0.conda
 https://repo.anaconda.com/pkgs/main/linux-64/pcre-8.45-h295c915_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/pixman-0.43.2-h59595ed_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2
@@ -109,13 +109,14 @@ https://conda.anaconda.org/conda-forge/linux-64/xcb-util-renderutil-0.3.9-h166bd
 https://conda.anaconda.org/conda-forge/linux-64/xcb-util-wm-0.4.1-h516909a_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.4-h0b41bf4_0.conda
 https://repo.anaconda.com/pkgs/main/linux-64/zstd-1.5.5-hc292b87_0.conda
+https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/attrs-23.2.0-pyh71513ae_0.conda
 https://repo.anaconda.com/pkgs/main/noarch/backcall-0.2.0-pyhd3eb1b0_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/bcrypt-4.1.2-py311h46250e7_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hd590300_1.conda
 https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py311hb755f60_1.conda
 https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/certifi-2024.2.2-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/certifi-2024.6.2-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.3.2-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda
 https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_0.conda
@@ -225,6 +226,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libsndfile-1.2.2-hc60ed4a_1.cond
 https://repo.anaconda.com/pkgs/main/linux-64/libwebp-1.2.4-h11a3e52_1.conda
 https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.7-pyhd8ed1ab_0.conda
 https://repo.anaconda.com/pkgs/main/linux-64/numpy-1.23.5-py311h5585df3_0.conda
+https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda
 https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_0.conda
@@ -249,6 +251,7 @@ https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/gstreamer-1.22.0-h25f0c4b_2.conda
 https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-6.0.0-h8e241bc_0.conda
 https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.5-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-7.1.0-hd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2023.12.1-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_0.conda

source-code/hydra/README.md

@@ -6,6 +6,7 @@ in the context of scientific software.
 It support configuration file handling, command line arguments, logging,
 multiruns and so on.
 
+
 ## What is it?
 
 1. `gen_rand.py`: Python script to write random numbers to standard
@@ -14,23 +15,50 @@ multiruns and so on.
    1. `config.yaml`: configuration file with the defaults.
    1. `distr/gauss.yaml`: configuration file for the Gaussian distirubtion.
    1. `distr/uniform.yaml`: configuration file for the uniform distirubtion.
+   1. `file_config.yaml`: configuration file with the output file name.
+1. `debug.py`: Python script that simply prints the configuration settings for
+   debugging purposes.
+
 
 ## How to use it?
+
 Run with configuratino file settings:
 ```bash
 $ ./gen_rand.py
 ```
+
 To increase the number of random values:
 ```bash
-$ ./gen_rand.py 10
+$ ./gen_rand.py  n=10
 ```
 
 To use a uniform distribution:
 ```bash
-$ ./gen_rand.py distr=uniform
+$ ./gen_rand.py  distr=uniform
 ```
 
 To use a uniform distribution between -1 and 0:
 ```bash
-$ ./gen_rand.py distr=uniform distr.a=-1.0 distr.b=0.0
+$ ./gen_rand.py  distr=uniform  distr.a=-1.0  distr.b=0.0
+```
+
+To use a different configuration file:
+```bash
+$ ./gen_rand.py  -cn file_config.yaml
+``` 
+or
+```bash
+$ ./gen_rand.py  --config-name=file_config.yaml
+```
+
+To perform multiple runs with different parameter values:
+```bash
+$ ./gen_rand.py -m distr=uniform,gauss
+``` 
+Note that if multiple parameters are to be varied, the Cartesian product of
+the values will be used.
+
+To view the configuration settings, e.g., for debugging:
+```bash 
+$ ./gen_rand.py  --cfg=job  distr=uniform
 ```

source-code/hydra/conf/config.yaml

@@ -1,4 +1,5 @@
-verbose: False
 n: 1
+file: false
 defaults:
-    - distr: gauss
+  - distr: gauss
+  - _self_

source-code/hydra/conf/distr/gauss.yaml

@@ -1,4 +1,3 @@
-distr:
-    name: gauss
-    mu: 0.0
-    sigma: 1.0
+name: gauss
+mu: 0.0
+sigma: 1.0

source-code/hydra/conf/distr/uniform.yaml

@@ -1,4 +1,3 @@
-distr:
-    name: uniform
-    a: 0.0
-    b: 1.0
+name: uniform
+a: 0.0
+b: 1.0

source-code/hydra/conf/file_config.yaml

@@ -0,0 +1,8 @@
+hydra:
+  job:
+    chdir: true
+n: 1
+file: output.txt
+defaults:
+  - distr: gauss
+  - _self_

source-code/hydra/debug.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+from omegaconf import DictConfig, OmegaConf
+import hydra
+
+@hydra.main(version_base=None, config_path='conf/', config_name='config')
+def my_app(cfg):
+    print(OmegaConf.to_yaml(cfg))
+
+if __name__ == '__main__':
+    my_app()

source-code/hydra/gen_rand.py

@@ -5,36 +5,34 @@
 import numpy as np
 import sys
 
-LOG = logging.getLogger(sys.argv[0])
 
-@hydra.main(config_path='conf/config.yaml')
+@hydra.main(version_base=None, config_path='conf/', config_name='config')
 def gen_rand(cfg):
-    if cfg.verbose:
-        print(cfg.pretty(), file=sys.stderr)
+    logger = logging.getLogger(__name__)
     if cfg.n <= 0:
-        LOG.error(f'negative number to generate {cfg.n}')
-        return 1
-    LOG.info(f'generating {cfg.n} random numbers')
-    LOG.info(f'using {cfg.distr.name} distribution')
+        logger.error(f'negative number to generate {cfg.n}')
+        sys.exit(1)
+    logger.info(f'generating {cfg.n} random numbers')
+    logger.info(f'using {cfg.distr.name} distribution')
     if cfg.distr.name == 'gauss':
-        LOG.info(f'mu={cfg.distr.mu}, sigma={cfg.distr.sigma}')
+        logger.info(f'mu={cfg.distr.mu}, sigma={cfg.distr.sigma}')
         if cfg.distr.sigma <= 0:
-            LOG.error(f'negative standard deviation {cfg.distr.sigma}')
-            return 1
+            logger.error(f'negative standard deviation {cfg.distr.sigma}')
+            sys.exit(1)
         numbers = np.random.normal(loc=cfg.distr.mu, scale=cfg.distr.sigma,
                                    size=(cfg.n,))
     elif cfg.distr.name == 'uniform':
-        LOG.info(f'a={cfg.distr.a}, b={cfg.distr.b}')
+        logger.info(f'a={cfg.distr.a}, b={cfg.distr.b}')
         if cfg.distr.a >= cfg.distr.b:
-            LOG.warning(f'lower bound exceed upper bound, '
+            logger.warning(f'lower bound exceed upper bound, '
                         f'{cfg.distr.a} >= {cfg.distr.b}')
         numbers = np.random.uniform(cfg.distr.a, cfg.distr.b, size=(cfg.n,))
-    LOG.info('starting output')
+    logger.info('starting output')
+    out = open(cfg.file, 'w') if cfg.file else sys.stdout
     for number in numbers:
-        print(number)
-    LOG.info('output done')
+        print(number, file=out)
+    logger.info('output done')
     return 0
 
 if __name__ == '__main__':
-    status = gen_rand()
-    sys.exit(status)
+    gen_rand()