fix(ena-submission): Use db url instead of host for connections, do n…

…ot hardcode db name (#2890)

* Use url instead of host, do not hardcode db name

* Rename to schema from ena-submission to schema_ena_deposition

.github/workflows/build-arm-images.yaml

@@ -24,11 +24,11 @@ jobs:
     with:
       build_arm: true
   trigger-ena-submission:
-    uses: ./.github/workflows/ena-submission-image.yml
+    uses: ./.github/workflows/ena-submission-image.yaml
     with:
       build_arm: true
   trigger-ena-submission-flyway:
-    uses: ./.github/workflows/ena-submission-flyway-image.yml
+    uses: ./.github/workflows/ena-submission-flyway-image.yaml
     with:
       build_arm: true
   trigger-keycloakify:

ena-submission/README.md

@@ -104,7 +104,7 @@ In our kubernetes pod we run flyway in a docker container, however when running
 You can then create the schema using the following command:
 
 ```sh
-flyway -user=postgres -password=unsecure -url=jdbc:postgresql://127.0.0.1:5432/loculus -schemas=ena-submission -locations=filesystem:./flyway/sql migrate
+flyway -user=postgres -password=unsecure -url=jdbc:postgresql://127.0.0.1:5432/loculus -schemas=ena_deposition_schema -locations=filesystem:./flyway/sql migrate
 ```
 
 If you want to test the docker image locally. It can be built and run using the commands:

ena-submission/config/config.yaml

@@ -2,7 +2,7 @@ backend_url: http://localhost:8079/
 keycloak_token_url: http://localhost:8083/realms/loculus/protocol/openid-connect/token
 db_username: postgres
 db_password: unsecure
-db_host: "127.0.0.1"
+db_url: "jdbc:postgresql://127.0.0.1:5432/loculus?options=-c%20search_path%3Dena_deposition_schema"
 organisms:
   cchf:
     ingest:

ena-submission/flyway/conf/flyway.conf

@@ -1,3 +1,3 @@
 flyway.locations=filesystem:/flyway/sql
-flyway.schemas=ena-submission
+flyway.schemas=ena_deposition_schema
 flyway.baselineOnMigrate=true

ena-submission/scripts/create_assembly.py

@@ -57,7 +57,7 @@ class Config:
     password: str
     db_username: str
     db_password: str
-    db_host: str
+    db_url: str
     db_name: str
     unique_project_suffix: str
     ena_submission_url: str
@@ -565,7 +565,7 @@ def create_assembly(
         config = Config(**relevant_config)
     logger.info(f"Config: {config}")
 
-    db_config = db_init(config.db_password, config.db_username, config.db_host)
+    db_config = db_init(config.db_password, config.db_username, config.db_url)
     slack_config = slack_conn_init(
         slack_hook_default=config.slack_hook,
         slack_token_default=config.slack_token,

ena-submission/scripts/create_project.py

@@ -51,7 +51,7 @@ class Config:
     password: str
     db_username: str
     db_password: str
-    db_host: str
+    db_url: str
     db_name: str
     unique_project_suffix: str
     ena_submission_url: str
@@ -389,7 +389,7 @@ def create_project(log_level, config_file, test=False, time_between_iterations=1
         config = Config(**relevant_config)
     logger.info(f"Config: {config}")
 
-    db_config = db_init(config.db_password, config.db_username, config.db_host)
+    db_config = db_init(config.db_password, config.db_username, config.db_url)
     slack_config = slack_conn_init(
         slack_hook_default=config.slack_hook,
         slack_token_default=config.slack_token,

ena-submission/scripts/create_sample.py

@@ -56,7 +56,7 @@ class Config:
     password: str
     db_username: str
     db_password: str
-    db_host: str
+    db_url: str
     db_name: str
     unique_project_suffix: str
     ena_submission_url: str
@@ -431,7 +431,7 @@ def create_sample(log_level, config_file, test=False, time_between_iterations=10
         config = Config(**relevant_config)
     logger.info(f"Config: {config}")
 
-    db_config = db_init(config.db_password, config.db_username, config.db_host)
+    db_config = db_init(config.db_password, config.db_username, config.db_url)
     slack_config = slack_conn_init(
         slack_hook_default=config.slack_hook,
         slack_token_default=config.slack_token,

ena-submission/scripts/get_ena_submission_list.py

@@ -33,7 +33,7 @@ class Config:
     ingest_pipeline_submitter: str
     db_username: str
     db_password: str
-    db_host: str
+    db_url: str
     slack_hook: str
     slack_token: str
     slack_channel_id: str
@@ -125,7 +125,7 @@ def get_ena_submission_list(log_level, config_file, output_file):
     db_config = db_init(
         db_password_default=config.db_password,
         db_username_default=config.db_username,
-        db_host_default=config.db_host,
+        db_url_default=config.db_url,
     )
 
     entries_to_submit = {}

ena-submission/scripts/submission_db_helper.py

@@ -1,4 +1,5 @@
 import os
+import re
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from enum import Enum
@@ -9,8 +10,24 @@
 from psycopg2.pool import SimpleConnectionPool
 
 
+def convert_jdbc_to_psycopg2(jdbc_url):
+    jdbc_pattern = r"jdbc:postgresql://(?P<host>[^:/]+)(?::(?P<port>\d+))?/(?P<dbname>[^?]+)"
+
+    match = re.match(jdbc_pattern, jdbc_url)
+
+    if not match:
+        msg = "Invalid JDBC URL format."
+        raise ValueError(msg)
+
+    host = match.group("host")
+    port = match.group("port") or "5432"  # Default to 5432 if no port is provided
+    dbname = match.group("dbname")
+
+    return f"postgresql://{host}:{port}/{dbname}"
+
+
 def db_init(
-    db_password_default: str, db_username_default: str, db_host_default: str
+    db_password_default: str, db_username_default: str, db_url_default: str
 ) -> SimpleConnectionPool:
     db_password = os.getenv("DB_PASSWORD")
     if not db_password:
@@ -20,18 +37,17 @@ def db_init(
     if not db_username:
         db_username = db_username_default
 
-    db_host = os.getenv("DB_HOST")
-    if not db_host:
-        db_host = db_host_default
+    db_url = os.getenv("DB_URL")
+    if not db_url:
+        db_url = db_url_default
 
+    db_dsn = convert_jdbc_to_psycopg2(db_url) + "?options=-c%20search_path%3Dena_deposition_schema"
     return SimpleConnectionPool(
         minconn=1,
         maxconn=2,  # max 7*2 connections to db allowed
-        dbname="loculus",
         user=db_username,
-        host=db_host,
         password=db_password,
-        options="-c search_path=ena-submission",
+        dsn=db_dsn,
     )
 
 

ena-submission/scripts/trigger_submission_to_ena.py

@@ -1,7 +1,6 @@
 # This script adds all approved sequences to the submission_table
 # - this should trigger the submission process.
 
-import base64
 import json
 import logging
 import time
@@ -34,7 +33,7 @@ class Config:
     organism: str
     db_username: str
     db_password: str
-    db_host: str
+    db_url: str
     github_url: str
 
 
@@ -94,7 +93,7 @@ def trigger_submission_to_ena(
         config = Config(**relevant_config)
     logger.info(f"Config: {config}")
 
-    db_config = db_init(config.db_password, config.db_username, config.db_host)
+    db_config = db_init(config.db_password, config.db_username, config.db_url)
 
     if input_file:
         # Get sequences to upload from a file

ena-submission/scripts/upload_external_metadata_to_loculus.py

@@ -41,7 +41,7 @@ class Config:
     ena_specific_metadata: list[str]
     db_username: str
     db_password: str
-    db_host: str
+    db_url: str
     slack_hook: str
     slack_token: str
     slack_channel_id: str
@@ -222,7 +222,7 @@ def upload_external_metadata(log_level, config_file, time_between_iterations=10)
         relevant_config = {key: full_config.get(key, []) for key in Config.__annotations__}
         config = Config(**relevant_config)
     logger.info(f"Config: {config}")
-    db_config = db_init(config.db_password, config.db_username, config.db_host)
+    db_config = db_init(config.db_password, config.db_username, config.db_url)
     slack_config = slack_conn_init(
         slack_hook_default=config.slack_hook,
         slack_token_default=config.slack_token,

kubernetes/loculus/templates/ena-submission-deployment.yaml

@@ -59,11 +59,11 @@ spec:
                 secretKeyRef:
                   name: service-accounts
                   key: externalMetadataUpdaterPassword
-            - name: DB_HOST
+            - name: DB_URL
               valueFrom:
                 secretKeyRef:
                   name: database
-                  key: host
+                  key: url
             - name: DB_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -152,11 +152,11 @@ spec:
                     secretKeyRef:
                       name: service-accounts
                       key: externalMetadataUpdaterPassword
-                - name: DB_HOST
+                - name: DB_URL
                   valueFrom:
                     secretKeyRef:
                       name: database
-                      key: host
+                      key: url
                 - name: DB_USERNAME
                   valueFrom:
                     secretKeyRef: