Add S3 integration tests

.github/workflows/s3-integration.yml

@@ -0,0 +1,84 @@
+# S3 (MinIO) integration tests.
+#
+# Kept as a separate workflow from linux-build.yml: it needs Docker to run a
+# MinIO server and exercises a much narrower slice of htslib (the hfile_s3
+# backend), so it shouldn't block or slow down the main build matrix.
+name: S3 integration (MinIO)
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  MINIO_ROOT_USER: minioadmin
+  MINIO_ROOT_PASSWORD: minioadmin
+  MINIO_TEST_BUCKET: htslib-test
+
+jobs:
+  minio-s3:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+      with:
+        submodules: true
+        persist-credentials: false
+
+    # The official MinIO images are no longer published, so this uses
+    # Chainguard's build, pinned by digest.
+    - name: Start MinIO
+      run: |
+        docker run -d --name minio-test -p 9000:9000 \
+          -e MINIO_ROOT_USER="$MINIO_ROOT_USER" \
+          -e MINIO_ROOT_PASSWORD="$MINIO_ROOT_PASSWORD" \
+          cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1 \
+          server /tmp/data
+
+    - name: Create test bucket
+      run: |
+        for i in $(seq 1 60); do
+          if curl -sf http://localhost:9000/minio/health/ready >/dev/null; then
+            curl -sf --aws-sigv4 aws:amz:us-east-1:s3 \
+              --user "$MINIO_ROOT_USER:$MINIO_ROOT_PASSWORD" \
+              -X PUT "http://localhost:9000/$MINIO_TEST_BUCKET"
+            exit
+          fi
+          sleep 1
+        done
+        echo "MinIO did not become ready in time" >&2
+        docker logs minio-test || true
+        exit 1
+
+    - name: Install build dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-suggests --no-install-recommends \
+          autoconf automake make gcc perl zlib1g-dev libbz2-dev liblzma-dev \
+          libcurl4-gnutls-dev libssl-dev
+
+    - name: Configure and build
+      run: |
+        autoreconf -i
+        ./configure --enable-werror --enable-plugins
+        make -j5
+
+    - name: Run S3 integration tests
+      env:
+        HTS_PATH: .
+        HTS_S3_HOST: localhost:9000
+        HTS_S3_ADDRESS_STYLE: path
+        AWS_ACCESS_KEY_ID: minioadmin
+        AWS_SECRET_ACCESS_KEY: minioadmin
+        HTS_ALLOW_UNENCRYPTED_AUTHORIZATION_HEADER: I understand the risks
+        HTSLIB_TEST_S3_BUCKET: htslib-test
+      run: make check-s3
+
+    - name: MinIO container logs
+      if: always()
+      run: docker logs minio-test || true
+
+    - name: Stop MinIO
+      if: always()
+      run: docker rm -f minio-test || true

.gitignore

@@ -59,6 +59,7 @@ shlib-exports-*.txt
 /test/sam
 /test/ref_cache/*.tmp.*
 /test/tabix/*.tmp.*
+/test/s3/*.tmp*
 /test/test_alloc
 /test/test-bcf-sr
 /test/test-bcf-translate
@@ -74,6 +75,7 @@ shlib-exports-*.txt
 /test/test_kstring
 /test/test_mod
 /test/test_hfile_libcurl
+/test/test_hfile_s3
 /test/test_nibbles
 /test/test-parse-reg
 /test/test_realn

Makefile

@@ -731,6 +731,18 @@ check test: all $(HTSCODECS_TEST_TARGETS)
 	  test/test_hfile_libcurl ; \
 	fi
 
+# S3 tests.  Not part of `check` as they need an S3-compatible server,
+# set via HTS_S3_HOST and friends (see .github/workflows/s3-integration.yml).
+# Both skip themselves if HTS_S3_HOST is not set.
+check-s3 test-s3: all test/test_hfile_s3
+	if test "x$(BUILT_PLUGINS)" != "x"; then \
+	  HTS_PATH=. ./test/with-shlib.sh test/test_hfile_s3 && \
+	  cd test/s3 && REF_PATH=: HTS_PATH=../.. ../with-shlib.sh ./test-s3.sh s3.tst ; \
+	else \
+	  test/test_hfile_s3 && \
+	  cd test/s3 && REF_PATH=: ./test-s3.sh s3.tst ; \
+	fi
+
 test/hts_endian: test/hts_endian.o
 	$(CC) $(LDFLAGS) -o $@ test/hts_endian.o $(LIBS)
 
@@ -831,6 +843,9 @@ test/test-bcf_set_variant_type: test/test-bcf_set_variant_type.o libhts.a
 test/test_hfile_libcurl: test/test_hfile_libcurl.o libhts.a
 	$(CC) $(LDFLAGS) -o $@ test/test_hfile_libcurl.o libhts.a $(LIBS) -lpthread
 
+test/test_hfile_s3: test/test_hfile_s3.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_hfile_s3.o libhts.a $(LIBS) -lpthread
+
 # Extra tests for bundled htscodecs
 test_htscodecs_rans4x8: htscodecs/tests/rans4x8
 	cd htscodecs/tests && srcdir=. && export srcdir && ./rans4x8.test
@@ -913,6 +928,7 @@ test/test-bcf-translate.o: test/test-bcf-translate.c config.h $(htslib_vcf_h)
 test/test_introspection.o: test/test_introspection.c config.h $(htslib_hts_h) $(htslib_hfile_h)
 test/test-bcf_set_variant_type.o: test/test-bcf_set_variant_type.c config.h $(htslib_hts_h) vcf.c
 test/test_hfile_libcurl.o: test/test_hfile_libcurl.c config.h $(htslib_hfile_h) $(htslib_hts_h) $(hts_internal_h)
+test/test_hfile_s3.o: test/test_hfile_s3.c config.h $(htslib_hfile_h)
 
 # Standalone target not added to $(BUILT_TEST_PROGRAMS) as some may not
 # have a compiler that compiles as C++ when given a .cpp source file.
@@ -1101,11 +1117,11 @@ distdir: htscodecs/htscodecs/version.h
 force:
 
 
-.PHONY: all check check-untracked clean distclean distdir force
+.PHONY: all check check-s3 check-untracked clean distclean distdir force
 .PHONY: install install-pkgconfig installdirs lib-shared lib-static
 .PHONY: maintainer-check maintainer-clean mostlyclean plugins
 .PHONY: print-config print-version show-version tags
-.PHONY: test test-shlib-exports test_thrash testclean
+.PHONY: test test-s3 test-shlib-exports test_thrash testclean
 .PHONY: clean-so install-so
 .PHONY: clean-cygdll install-cygdll
 .PHONY: clean-dll install-dll

test/s3/s3.tst

@@ -0,0 +1,89 @@
+#    Copyright (C) 2026 Peter Dowdy.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# First field:
+#   INIT = initialisation, not counted in testing
+#   P = expected to pass (zero return; expected output matches, if present)
+#   N = expected to return non-zero
+#   F = expected to fail
+#
+# Second field (P/N/F only):
+#   Filename of expected output.  If '.', output is not checked
+#
+# Rest:
+#   Command to execute.  $s3 is the S3 URL prefix for this run, and $bgzip,
+# $tabix, $htsfile, $test_view and $test_faidx are the paths to those programs.
+#
+# Expected outputs not already in test/ are made by running the same command
+# on a local copy, so each test checks that going via S3 changes nothing.
+# Indexes loaded for a remote file are cached in the current directory, so
+# S3 keys are named *.tmp.* and those caches removed first.
+
+INIT rm -f *.tmp.*
+
+# Raw copy up and back down
+P . $htsfile -C ../bgziptest.txt $s3/copy.tmp.txt
+P ../bgziptest.txt $htsfile -C $s3/copy.tmp.txt -
+N . $htsfile $s3/does-not-exist.tmp.bam
+
+# BGZF with a .gzi index, random access through the remote index
+INIT seq 1 20000 > seq.tmp.txt
+INIT $bgzip -i -o seq.tmp.txt.gz seq.tmp.txt
+INIT $bgzip -b 71088 -s 60 seq.tmp.txt.gz > seq.tmp.out
+P . $bgzip -i -o $s3/seq.tmp.txt.gz seq.tmp.txt
+P seq.tmp.out $bgzip -b 71088 -s 60 $s3/seq.tmp.txt.gz
+
+# BAM with CSI and BAI indexes, threaded read and write
+INIT $test_view -b -m 14 -x local.tmp.bam.csi -p local.tmp.bam ../index.sam
+INIT $test_view local.tmp.bam CHROMOSOME_II:2960-2970 > bam_region.tmp.out
+INIT $test_view local.tmp.bam > bam_all.tmp.out
+INIT $htsfile local.tmp.bam | cut -f 2 > htsfile.tmp.out
+P . $test_view -b -m 14 -x $s3/csi.tmp.bam.csi -p $s3/csi.tmp.bam ../index.sam
+P bam_region.tmp.out $test_view $s3/csi.tmp.bam CHROMOSOME_II:2960-2970
+P htsfile.tmp.out $htsfile $s3/csi.tmp.bam | cut -f 2
+P . $test_view -b -m 0 -x $s3/bai.tmp.bam.bai -p $s3/bai.tmp.bam ../index.sam
+P bam_region.tmp.out $test_view $s3/bai.tmp.bam CHROMOSOME_II:2960-2970
+P . $test_view -@ 4 -b -p $s3/threads.tmp.bam ../index.sam
+P bam_all.tmp.out $test_view -@ 4 $s3/threads.tmp.bam
+
+# CRAM with the reference and .crai index both on S3 (headers are skipped
+# as @SQ UR: records where the reference was read from)
+INIT $test_view -C -t ../ce.fa -x local.tmp.cram.crai -p local.tmp.cram ../index.sam
+INIT $test_view -i reference=../ce.fa local.tmp.cram CHROMOSOME_II:2960-2970 | grep -v "^@" > cram_region.tmp.out
+P . $htsfile -C ../ce.fa $s3/ce.tmp.fa
+P . $htsfile -C ../ce.fa.fai $s3/ce.tmp.fa.fai
+P . $test_view -C -t $s3/ce.tmp.fa -x $s3/ref.tmp.cram.crai -p $s3/ref.tmp.cram ../index.sam
+P cram_region.tmp.out $test_view -i reference=$s3/ce.tmp.fa $s3/ref.tmp.cram CHROMOSOME_II:2960-2970 | grep -v "^@"
+
+# VCF, BCF and BED with tabix indexes built and queried remotely
+P . $bgzip -o $s3/vcf.tmp.vcf.gz ../tabix/vcf_file.vcf
+P . $tabix -p vcf $s3/vcf.tmp.vcf.gz
+P ../tabix/vcf_file.1.3000151.out $tabix -D $s3/vcf.tmp.vcf.gz 1:3000151-3000151
+P . $test_view -b -m 14 -x $s3/bcf.tmp.bcf.csi -p $s3/bcf.tmp.bcf ../tabix/vcf_file.vcf
+P ../tabix/vcf_file.2.3199812.out $tabix -D $s3/bcf.tmp.bcf 2:3199812-3199812
+P . $bgzip -o $s3/bed.tmp.bed.gz ../tabix/bed_file.bed
+P . $tabix -p bed $s3/bed.tmp.bed.gz
+P ../tabix/bed_file.Y.100200.out $tabix -D $s3/bed.tmp.bed.gz Y:100200-100200
+
+# Bgzipped FASTA: .fai built on S3, then loaded with the .gzi
+INIT $test_faidx -c -i ../faidx/faidx.fa -f local.tmp.fai trailingblank1 > faidx.tmp.out
+P . $bgzip -i -o $s3/faidx.tmp.fa.gz ../faidx/faidx.fa
+P faidx.tmp.out $test_faidx -c -i $s3/faidx.tmp.fa.gz trailingblank1
+P faidx.tmp.out $test_faidx -i $s3/faidx.tmp.fa.gz trailingblank1

test/s3/test-s3.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+#    Copyright (C) 2026 Peter Dowdy.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# Runs s3.tst against the S3-compatible server given by HTS_S3_HOST.
+# Needs credentials in AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, and an
+# existing bucket named by HTSLIB_TEST_S3_BUCKET (default htslib-test).
+
+# Load in the test driver
+. ../simple_test_driver.sh
+
+if [ -z "$HTS_S3_HOST" ]
+then
+    echo "HTS_S3_HOST not set, skipping S3 tests"
+    exit 0
+fi
+
+echo "Testing S3..."
+
+s3="s3+http://${HTSLIB_TEST_S3_BUCKET:-htslib-test}/test-s3-$$-`date +%s`"
+bgzip="../../bgzip"
+tabix="../../tabix"
+htsfile="../../htsfile"
+test_view="../test_view"
+test_faidx="../test_faidx"
+
+test_driver $@
+
+exit $?