Updates for daily loading, kibo, sqlalchemy 2.

.github/workflows/python-package.yml

@@ -17,30 +17,35 @@ jobs:
             fail-fast: true
             matrix:
                 os: [ubuntu-latest]
-                python-version: ['3.9', '3.10']
-                numpy-version: ['<1.23']
-                astropy-version: ['<5.1', '<6.0']
+                python-version: ['3.10']  # , '3.11'] The 3.11 tests take a long time, so skip for now.
+                numpy-version: ['<1.23']  # , '<2.0'] Need to find a scipy version that works for 3.11 and pre-2.0 numpy
+                astropy-version: ['<6.1', '<7.0']
 
         env:
-            DESIUTIL_VERSION: '3.3.0'
-            DESISPEC_VERSION: '0.56.2'
+            # Versions correspond to desimodules/24.6
+            DESIUTIL_VERSION: '3.4.2'
+            DESISPEC_VERSION: '0.63.7'
+            DESITARGET_VERSION: '2.7.0'
+            DESIMODEL_VERSION: '0.19.1'
 
         steps:
             - name: Checkout code
-              uses: actions/checkout@v2
+              uses: actions/checkout@v4
               with:
                 fetch-depth: 0
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v5
               with:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
               run: |
                 python -m pip install --upgrade pip setuptools wheel
                 python -m pip install git+https://github.com/desihub/desiutil.git@${DESIUTIL_VERSION}#egg=desiutil
                 python -m pip install git+https://github.com/desihub/desispec.git@${DESISPEC_VERSION}#egg=desispec
+                python -m pip install git+https://github.com/desihub/desitarget.git@${DESITARGET_VERSION}#egg=desitarget
+                python -m pip install git+https://github.com/desihub/desimodel.git@${DESIMODEL_VERSION}#egg=desimodel
                 python -m pip install --upgrade "numpy${{ matrix.numpy-version }}"
-                python -m pip install --upgrade "astropy${{ matrix.astropy-version }}"
+                python -m pip install --upgrade "astropy${{ matrix.astropy-version }}" 'scipy<1.13' fitsio numba
                 python -m pip install --editable .[test]
             - name: Run the test
               run: pytest
@@ -54,28 +59,32 @@ jobs:
                 os: [ubuntu-latest]
                 python-version: ['3.10']
                 numpy-version: ['<1.23']
-                astropy-version: ['<5.1']
+                astropy-version: ['<6.1']
 
         env:
-            DESIUTIL_VERSION: '3.3.0'
-            DESISPEC_VERSION: '0.56.2'
+            DESIUTIL_VERSION: '3.4.2'
+            DESISPEC_VERSION: '0.63.7'
+            DESITARGET_VERSION: '2.7.0'
+            DESIMODEL_VERSION: '0.19.1'
 
         steps:
             - name: Checkout code
-              uses: actions/checkout@v2
+              uses: actions/checkout@v4
               with:
                 fetch-depth: 0
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v5
               with:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
               run: |
                 python -m pip install --upgrade pip setuptools wheel
                 python -m pip install git+https://github.com/desihub/desiutil.git@${DESIUTIL_VERSION}#egg=desiutil
                 python -m pip install git+https://github.com/desihub/desispec.git@${DESISPEC_VERSION}#egg=desispec
+                python -m pip install git+https://github.com/desihub/desitarget.git@${DESITARGET_VERSION}#egg=desitarget
+                python -m pip install git+https://github.com/desihub/desimodel.git@${DESIMODEL_VERSION}#egg=desimodel
                 python -m pip install --upgrade "numpy${{ matrix.numpy-version }}"
-                python -m pip install --upgrade "astropy${{ matrix.astropy-version }}"
+                python -m pip install --upgrade "astropy${{ matrix.astropy-version }}" 'scipy<1.13' fitsio numba
                 python -m pip install --editable .[coverage]
             - name: Run the test with coverage
               run: pytest --cov
@@ -96,11 +105,11 @@ jobs:
 
         steps:
             - name: Checkout code
-              uses: actions/checkout@v2
+              uses: actions/checkout@v4
               with:
                 fetch-depth: 0
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v5
               with:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
@@ -117,15 +126,15 @@ jobs:
                 os: [ubuntu-latest]
                 python-version: ['3.10']
         env:
-            DESIUTIL_VERSION: '3.3.0'
+            DESIUTIL_VERSION: '3.4.2'
 
         steps:
             - name: Checkout code
-              uses: actions/checkout@v2
+              uses: actions/checkout@v4
               with:
                 fetch-depth: 0
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v5
               with:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
@@ -148,16 +157,16 @@ jobs:
 
         steps:
             - name: Checkout code
-              uses: actions/checkout@v2
+              uses: actions/checkout@v4
               with:
                 fetch-depth: 0
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v5
               with:
                 python-version: ${{ matrix.python-version }}
             - name: Install Python dependencies
               run: python -m pip install --upgrade pip setuptools wheel pycodestyle
-            - name: Test the style; failures are allowed
-              # This is equivalent to an allowed falure.
+            - name: Test the style
+              # Uncomment continue-on-error to allow failures.
               # continue-on-error: true
               run: pycodestyle --count py/specprodDB

.gitignore

@@ -110,6 +110,7 @@ ENV/
 env.bak/
 venv.bak/
 .vscode/
+spin/db/docker-env.txt
 
 # Spyder project settings
 .spyderproject

bin/load_specprod_tile

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# See top-level LICENSE.rst file for Copyright information
+from specprodDB.tile import main
+from sys import exit
+exit(main())

bin/patch_specprod

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# See top-level LICENSE.rst file for Copyright information
+from specprodDB.patch import main
+from sys import exit
+exit(main())

bin/submit_load_specprod_db.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Submit load_specprod_db jobs in sequence.
+#
+function usage() {
+    local execName=$(basename $0)
+    (
+        echo "${execName} [-h] [-s SHELL] [-t] [-v] SPECPROD"
+        echo ""
+        echo "Submit load_specprod_db jobs in sequence."
+        echo ""
+        echo "       -h = Print help message and exit."
+        echo " -s SHELL = Scripts have extension SHELL, default 'sh'."
+        echo "       -t = Test mode.  Do not make any changes.  Implies -v."
+        echo "       -v = Verbose mode. Print lots of extra information."
+        echo " SPECPROD = The specprod jobs to submit, e.g. 'iron'."
+    ) >&2
+}
+#
+# Command-line options.
+#
+shell=sh
+test=/usr/bin/false
+verbose=/usr/bin/false
+while getopts hs:tv argname; do
+    case ${argname} in
+        h) usage; exit 0 ;;
+        s) shell=${OPTARG} ;;
+        t) test=/usr/bin/true; verbose=/usr/bin/true ;;
+        v) verbose=/usr/bin/true ;;
+        *) usage; exit 1 ;;
+    esac
+done
+shift $((OPTIND - 1))
+#
+# Get the specprod value.
+#
+if [[ $# > 0 ]]; then
+    specprod=$1
+else
+    echo "ERROR: SPECPROD must be set!" >&2
+    exit 1
+fi
+#
+# Loop over jobs.
+#
+job_id=0
+for stage in exposures photometry targetphot target redshift fiberassign; do
+    if [[ "${stage}" == "exposures" ]]; then
+        dependency=''
+    else
+        dependency="--dependency=afterok:${job_id}"
+    fi
+    ${verbose} && echo "DEBUG: sbatch --parsable ${dependency} load_specprod_db_${specprod}_${stage}.${shell}"
+    if ${test}; then
+        job_id=$(( job_id + 1 ))
+    else
+        job_id=$(sbatch --parsable ${dependency} load_specprod_db_${specprod}_${stage}.${shell})
+    fi
+done

bin/submit_load_specprod_tile.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# Submit load_specprod_tile jobs in a chained sequence.
+#
+function usage() {
+    local execName=$(basename $0)
+    (
+        echo "${execName} [-h] [-j N] [-s SHELL] [-t] [-v]"
+        echo ""
+        echo "Submit load_specprod_tile jobs in sequence."
+        echo ""
+        echo "       -h = Print help message and exit."
+        echo "     -j N = Only submit N jobs."
+        echo " -s SHELL = Scripts have extension SHELL, default 'sh'."
+        echo "       -t = Test mode.  Do not make any changes.  Implies -v."
+        echo "       -v = Verbose mode. Print lots of extra information."
+    ) >&2
+}
+#
+# Command-line options.
+#
+n_jobs=0
+shell=sh
+test=/usr/bin/false
+verbose=/usr/bin/false
+while getopts hj:s:tv argname; do
+    case ${argname} in
+        h) usage; exit 0 ;;
+        j) n_jobs=${OPTARG} ;;
+        s) shell=${OPTARG} ;;
+        t) test=/usr/bin/true; verbose=/usr/bin/true ;;
+        v) verbose=/usr/bin/true ;;
+        *) usage; exit 1 ;;
+    esac
+done
+shift $((OPTIND - 1))
+#
+# Find the first tile and submit it.
+#
+first_tile=$(grep -- --overwrite load_specprod_tile_*.${shell} | cut -d: -f1)
+if [[ -n "${first_tile}" ]]; then
+    ${verbose} && echo "DEBUG: sbatch --parsable ${first_tile}"
+    if ${test}; then
+        job_id=1
+    else
+        job_id=$(sbatch --parsable ${first_tile})
+    fi
+    n_submitted=1
+else
+    echo "INFO: No first tile found, assuming it has already been loaded."
+    job_id=0
+    n_submitted=0
+fi
+#
+# Find the last tile.
+#
+last_tile=$(grep -- --primary load_specprod_tile_*.${shell} | cut -d: -f1)
+#
+# Loop over tiles.
+#
+for tile in load_specprod_tile_*.${shell}; do
+    if [[ "${tile}" == "${first_tile}" ]]; then
+        ${verbose} && echo "DEBUG: First tile, ${first_tile}, already submitted."
+    elif [[ "${tile}" == "${last_tile}" ]]; then
+        ${verbose} && echo "DEBUG: Last tile, ${last_tile}, will be skipped."
+    else
+        if [[ ${job_id} == 0 ]]; then
+            dependency=''
+        else
+            dependency="--dependency=afterok:${job_id}"
+        fi
+        ${verbose} && echo "DEBUG: sbatch --parsable ${dependency} ${tile}"
+        if ${test}; then
+            job_id=$(( job_id + 1 ))
+        else
+            job_id=$(sbatch --parsable ${dependency} ${tile})
+        fi
+        n_submitted=$(( n_submitted + 1 ))
+        if (( n_submitted == n_jobs )); then
+            echo "INFO: ${n_jobs} have been submitted, exiting."
+            exit 0
+        fi
+    fi
+done
+${verbose} && echo "DEBUG: sbatch --dependency=afterok:${job_id} ${last_tile}"
+if ${test}; then
+    :
+else
+    sbatch --dependency=afterok:${job_id} ${last_tile}
+fi

doc/api.rst

@@ -11,5 +11,11 @@ specprodDB API
 .. automodule:: specprodDB.load
     :members:
 
+.. automodule:: specprodDB.patch
+    :members:
+
+.. automodule:: specprodDB.tile
+    :members:
+
 .. automodule:: specprodDB.util
     :members:

doc/changes.rst

@@ -5,20 +5,25 @@ Change Log
 1.3.0 (unreleaded)
 ------------------
 
-*Planned*: Support loading ongoing daily reductions.
+*Planned*: Support loading ongoing daily reductions, in particular, updates
+to tiles already in the database.
 
 1.2.0 (unreleased)
 ------------------
 
 This release corresponds to loading the ``guadalupe`` and ``iron`` spectroscopic
-productions in preparation for DR1. See also PR `#12`_. Some specific changes:
+productions in preparation for DR1. See also PR `#12`_ and `#14`_. Some specific changes:
 
 * Photometry and target data are split among several files instead of a monolithic file.
-* Add ``DESINAME`` column to redshift tabled.
+* Add ``DESINAME`` column to redshift tables.
 * Move version configuration to a configuration file.
 * Default values for targeting bits when they are absent from target files.
+* Support SQLAlchemy 2.
+* ORM objects know how to "load themselves".
+* Support for tile-based loading, needed to load ``daily`` reductions.
 
 .. _`#12`: https://github.com/desihub/specprod-db/pull/12
+.. _`#14`: https://github.com/desihub/specprod-db/pull/14
 
 1.1.0 (2023-06-09)
 ------------------

doc/conf.py

@@ -78,7 +78,7 @@
 # some external dependencies are not met at build time and break the
 # building process.
 autodoc_mock_imports = []
-for missing in ('desiutil', 'desispec', 'numpy', 'astropy', 'pytz', 'sqlalchemy'):
+for missing in ('desiutil', 'desispec', 'desitarget', 'numpy', 'astropy', 'pytz', 'sqlalchemy'):
     try:
         foo = import_module(missing)
     except ImportError: