Emulate samtools et al if these tools are not available

Add devtools/emulate-tools.py script that, when invoked as samtools,
bcftools, bgzip, or tabix, emulates that tool's behaviour by using
pysam's facilities. The script implements all functionality used in
the test suite (and in particular when creating test data in
tests/*_data/Makefile).

devtools/install-prerequisites.sh installs symlinks to this script
on platforms where samtools/bcftools/tabix packages are not available.

Add a new "don't disturb stdout" redirection mode to _pysam_dispatch(),
for use by the script. For now this is invoked via catch_stdout=None,
but we won't document this facility until it has bedded in.

Author: jmarshall

devtools/emulate-tools.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""
+This script can be symlinked to samtools, bcftools, bgzip, or tabix.
+When invoked under one of those names, it will emulate that tool's
+behaviour by using pysam's facilities.
+"""
+
+import argparse
+import gzip
+import os
+import sys
+import tempfile
+
+import pysam
+
+command = os.path.basename(sys.argv[0])
+
+if command in ("samtools", "bcftools"):
+    if len(sys.argv) > 1:
+        try:
+            tool = pysam.utils.PysamDispatcher(command, sys.argv[1])
+            tool(*sys.argv[2:], catch_stdout=None)
+            print(tool.stderr, end="", file=sys.stderr)
+        except pysam.utils.SamtoolsError as e:
+            sys.exit(f"emulate-tools.py: {e}")
+
+    else:
+        version = getattr(pysam.version, f"__{command}_version__")
+        print(f"Program: {command}\nVersion: {version}", file=sys.stderr)
+
+else:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-c", "--stdout", action="store_true")
+    parser.add_argument("-d", "--decompress", action="store_true")
+    parser.add_argument("-f", "--force", action="store_true")
+    parser.add_argument("-p", "--preset")
+    parser.add_argument("input_file", nargs="?")
+    opt = parser.parse_args()
+
+    if command == "bgzip":
+        if opt.decompress:
+            with gzip.open(sys.stdin.buffer, "rb") as f:
+                sys.stdout.buffer.write(f.read())
+
+        elif opt.input_file and opt.stdout:
+            pysam.tabix_compress(opt.input_file, "-", force=True)
+
+        else:
+            f = tempfile.NamedTemporaryFile(delete=False)
+            f.write(sys.stdin.buffer.read())
+            f.close()
+            pysam.tabix_compress(f.name, "-", force=True)
+            os.remove(f.name)
+
+    elif command == "tabix":
+        pysam.tabix_index(opt.input_file, preset=opt.preset, force=opt.force)
+
+    else:
+        sys.exit(f"emulate-tools.py: unknown command {command!r}")

devtools/install-prerequisites.sh

@@ -12,6 +12,7 @@ elif test -x /usr/bin/yum; then
     else
         echo Installing non-test prerequisites via yum...
         yum -y install zlib-devel bzip2-devel xz-devel curl-devel openssl-devel
+        emulate=yes
     fi
 
 elif test -d /etc/dpkg; then
@@ -23,6 +24,7 @@ elif test -x /sbin/apk; then
     echo Installing non-test prerequisites via apk...
     apk update
     apk add zlib-dev bzip2-dev xz-dev curl-dev openssl-dev
+    emulate=yes
 
 elif test -x ${HOMEBREW_PREFIX-/usr/local}/bin/brew; then
     echo Installing prerequisites via brew...
@@ -32,3 +34,14 @@ elif test -x ${HOMEBREW_PREFIX-/usr/local}/bin/brew; then
 else
     echo No package manager detected
 fi
+
+if test -n "$emulate" && test $# -ge 2; then
+    emulator=$1
+    bindir=$2
+    echo Creating symlinks to $emulator in $bindir...
+    mkdir -p $bindir
+    ln -s $emulator $bindir/samtools
+    ln -s $emulator $bindir/bcftools
+    ln -s $emulator $bindir/bgzip
+    ln -s $emulator $bindir/tabix
+fi

pyproject.toml

@@ -22,7 +22,7 @@ requires = ["setuptools>=59.0", "Cython>=0.29.12,<4"]
 build-backend = "setuptools.build_meta:__legacy__"
 
 [tool.cibuildwheel]
-before-all = "{project}/devtools/install-prerequisites.sh"
+before-all = "{project}/devtools/install-prerequisites.sh {project}/devtools/emulate-tools.py /usr/local/bin"
 # Necessary until we build libhts.a out-of-tree from within build_temp
 before-build = "make -C {project}/htslib distclean"
 

pysam/libcutils.pyx

@@ -19,7 +19,7 @@ from libc.stdint cimport INT32_MAX, int32_t
 from libc.stdio cimport fprintf, stderr, fflush
 from libc.stdio cimport stdout as c_stdout
 from posix.fcntl cimport open as c_open, O_WRONLY, O_CREAT, O_TRUNC
-from posix.unistd cimport SEEK_SET, SEEK_CUR, SEEK_END
+from posix.unistd cimport dup as c_dup, SEEK_SET, SEEK_CUR, SEEK_END, STDOUT_FILENO
 
 from pysam.libcsamtools cimport samtools_dispatch, samtools_set_stdout, samtools_set_stderr, \
     samtools_close_stdout, samtools_close_stderr, samtools_set_stdout_fn
@@ -361,7 +361,8 @@ def _pysam_dispatch(collection,
     else:
         samtools_set_stdout_fn("-")
         bcftools_set_stdout_fn("-")
-        stdout_h = c_open(b"/dev/null", O_WRONLY)
+        if catch_stdout is None: stdout_h = c_dup(STDOUT_FILENO)
+        else: stdout_h = c_open(b"/dev/null", O_WRONLY)
 
     # setup the function call to samtools/bcftools main
     cdef char ** cargs