feat(ena-submission): Create insdc submission service

.github/workflows/build-arm-images.yaml

@@ -23,6 +23,14 @@ jobs:
     uses: ./.github/workflows/ingest-image.yml
     with:
       build_arm: true
+  trigger-ena-submission:
+    uses: ./.github/workflows/ena-submission-image.yml
+    with:
+      build_arm: true
+  trigger-ena-submission-flyway:
+    uses: ./.github/workflows/ena-submission-flyway-image.yml
+    with:
+      build_arm: true
   trigger-keycloakify:
     uses: ./.github/workflows/keycloakify-image.yml
     with:

.github/workflows/ena-submission-flyway-image.yaml

@@ -0,0 +1,69 @@
+name: ena-submission-flyway-image
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+    inputs:
+      build_arm:
+        type: boolean
+        description: "Build for ARM as well"
+        default: false
+        required: true
+  workflow_call:
+    inputs:
+      build_arm:
+        type: string
+        description: "Build for ARM as well"
+        default: "false"
+        required: true
+env:
+  DOCKER_IMAGE_NAME: ghcr.io/loculus-project/ena-submission-flyway
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}
+  sha: ${{ github.event.pull_request.head.sha || github.sha }}
+defaults:
+  run:
+    working-directory: ./ena-submission/flyway
+concurrency:
+  group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-ena-submission-flyway-${{github.event.inputs.build_arm}}
+  cancel-in-progress: true
+jobs:
+  ena-submission-flyway-image:
+    name: Build ena-submission-flyway Docker Image # Don't change: Referenced by .github/workflows/update-argocd-metadata.yml
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      packages: write
+      checks: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Shorten sha
+        run: echo "sha=${sha::7}" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker metadata
+        id: dockerMetadata
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_IMAGE_NAME }}
+          tags: |
+            type=raw,value=${{ env.BRANCH_NAME }}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+            type=raw,value=commit-${{ env.sha }}
+            type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
+      - name: Build and push image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./ena-submission/flyway
+          push: true
+          tags: ${{ steps.dockerMetadata.outputs.tags }}
+          platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}

.github/workflows/ena-submission-image.yaml

@@ -0,0 +1,87 @@
+name: ena-submission-image
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+    inputs:
+      build_arm:
+        type: boolean
+        description: "Build for ARM as well"
+        default: false
+        required: false
+  workflow_call:
+    inputs:
+      build_arm:
+        type: boolean
+        description: "Build for ARM as well"
+        default: false
+        required: false
+env:
+  DOCKER_IMAGE_NAME: ghcr.io/loculus-project/ena-submission
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}
+  sha: ${{ github.event.pull_request.head.sha || github.sha }}
+concurrency:
+  group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-ena-submission-${{github.event.inputs.build_arm}}
+  cancel-in-progress: true
+jobs:
+  ena-submission-image:
+    name: Build ena-submission Docker Image # Don't change: Referenced by .github/workflows/update-argocd-metadata.yml
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      packages: write
+      checks: read
+    steps:
+      - name: Shorten sha
+        run: echo "sha=${sha::7}" >> $GITHUB_ENV
+      - uses: actions/checkout@v4
+      - name: Generate files hash
+        id: files-hash
+        run: |
+          DIR_HASH=$(echo -n ${{ hashFiles('ena-submission/**', '.github/workflows/ena-submission-image.yml') }})
+          echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV
+      - name: Setup Docker metadata
+        id: dockerMetadata
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_IMAGE_NAME }}
+          tags: |
+            type=raw,value=${{ env.DIR_HASH }}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+            type=raw,value=${{ env.BRANCH_NAME }}
+            type=raw,value=commit-${{ env.sha }}
+            type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check if image exists
+        id: check-image
+        run: |
+          EXISTS=$(docker manifest inspect ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
+          echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push image if input files changed
+        if: env.CACHE_HIT == 'false'
+        uses: docker/build-push-action@v6
+        with:
+          context: ./ena-submission
+          push: true
+          tags: ${{ steps.dockerMetadata.outputs.tags }}
+          cache-from: type=gha,scope=ena-submission-${{ github.ref }}
+          cache-to: type=gha,mode=max,scope=ena-submission-${{ github.ref }}
+          platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
+      - name: Retag and push existing image if cache hit
+        if: env.CACHE_HIT == 'true'
+        run: |
+          TAGS=(${{ steps.dockerMetadata.outputs.tags }})
+          for TAG in "${TAGS[@]}"; do
+            docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }}
+          done

backend/src/main/kotlin/org/loculus/backend/config/SecurityConfig.kt

@@ -54,7 +54,6 @@ class SecurityConfig {
 
     private val endpointsForExternalMetadataUpdater = arrayOf(
         "/*/submit-external-metadata",
-        "/*/get-released-data",
     )
 
     private val getEndpointsThatArePublic = arrayOf(
@@ -87,8 +86,10 @@ class SecurityConfig {
             auth.requestMatchers(HttpMethod.GET, *getEndpointsThatArePublic).permitAll()
             auth.requestMatchers(HttpMethod.OPTIONS).permitAll()
             auth.requestMatchers(*endpointsForPreprocessingPipeline).hasAuthority(PREPROCESSING_PIPELINE)
+            auth.requestMatchers(
+                *endpointsForExternalMetadataUpdater,
+            ).hasAuthority(EXTERNAL_METADATA_UPDATER)
             auth.requestMatchers(*endpointsForGettingReleasedData).hasAuthority(GET_RELEASED_DATA)
-            auth.requestMatchers(*endpointsForExternalMetadataUpdater).hasAuthority(EXTERNAL_METADATA_UPDATER)
             auth.requestMatchers(*debugEndpoints).hasAuthority(SUPER_USER)
             auth.anyRequest().authenticated()
         }

backend/src/test/kotlin/org/loculus/backend/controller/RequestAuthorization.kt

@@ -17,7 +17,7 @@ val jwtForDefaultUser = generateJwtFor(DEFAULT_USER_NAME)
 val jwtForProcessingPipeline = generateJwtFor("preprocessing_pipeline", listOf(PREPROCESSING_PIPELINE))
 val jwtForGetReleasedData = generateJwtFor("silo_import_job", listOf(GET_RELEASED_DATA))
 val jwtForExternalMetadataUpdatePipeline =
-    generateJwtFor("external_metadata_updater", listOf(EXTERNAL_METADATA_UPDATER))
+    generateJwtFor("external_metadata_updater", listOf(EXTERNAL_METADATA_UPDATER, GET_RELEASED_DATA))
 val jwtForSuperUser = generateJwtFor(SUPER_USER_NAME, listOf(SUPER_USER))
 
 fun generateJwtFor(username: String, roles: List<String> = emptyList()): String = Jwts.builder()

deploy.py

@@ -90,6 +90,11 @@
 helm_parser.add_argument(
     "--enableIngest", action="store_true", help="Include deployment of ingest pipelines"
 )
+helm_parser.add_argument(
+    "--enableEnaSubmission",
+    action="store_true",
+    help="Include deployment of ena submission pipelines",
+)
 helm_parser.add_argument(
     "--values", help="Values file for helm chart", default=HELM_VALUES_FILE
 )
@@ -234,6 +239,9 @@ def handle_helm():
     if not args.enableIngest:
         parameters += ["--set", "disableIngest=true"]
 
+    if not args.enableEnaSubmission:
+        parameters += ["--set", "disableEnaSubmission=true"]
+
     if get_codespace_name():
         parameters += ["--set", "codespaceName=" + get_codespace_name()]
 
@@ -306,7 +314,7 @@ def generate_configs(from_live=False):
         from_live,
         ingest_configout_path,
     )
-
+    
     prepro_configmap_path = TEMP_DIR / "preprocessing-config.yaml"
     prepro_template_path = "templates/loculus-preprocessing-config.yaml"
     prepro_configout_path = TEMP_DIR / "preprocessing-config.yaml"

ena-submission/.gitignore

@@ -0,0 +1,2 @@
+.snakemake/
+results/

ena-submission/.mambarc

@@ -0,0 +1,6 @@
+channels:
+  - conda-forge
+  - bioconda
+repodata_use_zst: true
+channel_priority: strict
+download_threads: 20

ena-submission/Dockerfile

@@ -0,0 +1,15 @@
+FROM mambaorg/micromamba:1.5.8
+
+COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/env.yaml
+COPY --chown=$MAMBA_USER:$MAMBA_USER .mambarc /tmp/.mambarc
+
+RUN micromamba config set extract_threads 1 \
+    && micromamba install -y -n base -f /tmp/env.yaml --rc-file /tmp/.mambarc \
+    && micromamba clean --all --yes
+
+# Set the environment variable to activate the conda environment
+ARG MAMBA_DOCKERFILE_ACTIVATE=1
+
+COPY --chown=$MAMBA_USER:$MAMBA_USER . /package
+
+WORKDIR /package