fix(deploy): htsget networking (#102)

* fix(deploy): bypass proxy when using htsget with compose minio

* refactor(nginx): use recommended header opts for minio

* docs(deploy): note on aliasing minio host

* docs: release instructions

* docs(deploy): typos

* fix(deploy): forward s3 host header through nginx

* docs(deploy): alt opt for htsget x minio networking

* docs(deploy): section on htsget-minio addressing

* docs(deploy): s3 bucket style in example .env

* refactor(compose): scheme always inside URL env vars

Author: cmdoret

deploy/.example.env

@@ -5,12 +5,13 @@ MINIO_SERVER_ACCESS_KEY="user"
 MINIO_SERVER_SECRET_KEY="pass"
 S3_BUCKET=modos-demo
 MINIO_DEFAULT_BUCKET="${S3_BUCKET}:public"
+# defines s3 url format (host/bucket vs bucket.host)
+S3_ADDRESSING_STYLE="auto" # one of auto, path, virtual
 
 # HTSGET
 HTSGET_DATA_DIR="/data"
 
 # PUBLIC
-#URL_PREFIX="https://modos.example.com"
-URL_PREFIX="http://localhost"
-S3_PUBLIC_URL="${URL_PREFIX}/s3"
-HTSGET_PUBLIC_URL="${URL_PREFIX}/htsget"
+# URL_PREFIX="https://modos.example.org"
+# S3_PUBLIC_URL="${URL_PREFIX}/s3"
+# HTSGET_PUBLIC_URL="${URL_PREFIX}/htsget"

deploy/README.md

@@ -12,7 +12,7 @@ The modos-server is meant to provide remote access to the MODOs. Currently, it c
 - [x] list available MODO
 - [x] return their metadata
 - [x] expose a MODO directly as a client-accessible S3 bucket / folder
-- [ ] stream CRAM slices CRAM using htsget
+- [x] stream CRAM slices using htsget
 - [ ] manage authentication and access control
 
 The MODOs are stored in an s3 (minio) bucket, and an htsget server is deployed alongside the modos-server to handle slicing and streaming of CRAM files. A REST API is exposed to the client to interact with the remote MODOs.
@@ -79,3 +79,17 @@ mv .example.env .env
 docker compose up --build
 # docker compose automatically reads the .env file
 ```
+
+### Streaming with minio
+
+There are two options to use htsget streaming with the minio embedded in the compose setup:
+
+1. Either manually create a host mapping from the minio service to localhost:
+> `echo "127.0.0.1 minio" >> /etc/hosts`
+
+2. Or set `S3_PUBLIC_URL=http://<LOCAL-IP>:9000` where `<LOCAL-IP>` is your local IP address (find it using hostname -I).
+
+These steps are not needed when using an external S3 server, in which case `S3_PUBLIC_URL` can just be set to the external S3 endpoint.
+
+> [!NOTE]
+> These steps are needed because the S3 host must be available under the same name to both the client and htsget. This is because the canonical URI (incl. hostname) is used to [derive s3 signature keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html).

deploy/docker-compose.yml

@@ -15,7 +15,7 @@ services:
     volumes:
       - ./nginx/default.conf.template:/etc/nginx/templates/default.conf.template:ro
     environment:
-      - S3_PUBLIC_URL=${S3_PUBLIC_URL:-http://localhost/s3}
+      - S3_PUBLIC_URL=${S3_PUBLIC_URL:-http://localhost/s3/}
       - S3_LOCAL_URL=${S3_lOCAL_URL:-http://minio:9000}
       - HTSGET_PUBLIC_URL=${HTSGET_PUBLIC_URL:-http://localhost/htsget}
       - HTSGET_LOCAL_URL=${HTSGET_LOCAL_URL:-http://htsget:8080}
@@ -27,9 +27,8 @@ services:
 
   minio:
     image: docker.io/bitnami/minio:2024
-    expose:
-      - "9000" # api
     ports:
+      - "9000:9000" # api
       - "9001:9001" # console
     volumes:
       - minio-data:/bitnami/minio/data
@@ -56,7 +55,7 @@ services:
     networks:
       - modos-network
     environment:
-      - ENDPOINT=${S3_LOCAL_URL:-http://localhost/s3}
+      - ENDPOINT=${S3_PUBLIC_URL:-http://minio:9000}
       - BUCKET=${S3_BUCKET:-modos-demo}
       - S3_ADDRESSING_STYLE=${S3_ADDRESSING_STYLE:-auto}
       - AWS_ACCESS_KEY_ID=${MINIO_ROOT_USER:-minio}

deploy/nginx/default.conf.template

@@ -2,7 +2,10 @@
 
 server {
   listen 80 ;
-  client_max_body_size 0 ;
+  ignore_invalid_headers off;
+  client_max_body_size 0;
+  proxy_buffering off;
+  proxy_request_buffering off;
 
   # Inform client about public URLs
   location = / {
@@ -18,6 +21,16 @@ server {
   location /s3 {
     rewrite ^/s3/(.*) /$1 break ;
     proxy_pass ${S3_LOCAL_URL} ;
+    # More details on minio x nginx options at:
+    # https://min.io/docs/minio/linux/integrations/setup-nginx-proxy-with-minio.html
+    proxy_set_header Host $http_host;
+    proxy_pass_request_headers off;
+    proxy_connect_timeout 300;
+    # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    chunked_transfer_encoding off;
+
   }
   # Genomic file streaming with htsget
   location /htsget {

docs/development_guide.md

@@ -19,3 +19,10 @@ you can enter a development shell with all requirements installed by doing:
 ```shell
 nix develop ./nix#default
 ```
+
+## Making a release
+
+Releases are deployed to pypi.org through github actions.
+To create a new release, create a PR named "chore: bump to X.Y.Z" where X.Y.Z is the new version. In the PR upgrade versions in the repo (sphinx config and pyproject.toml).
+
+Once the PR is merged, a release can be created through the github UI on the merge commit. This will trigger corresponding release builds on PyPI and ghcr.io.