Reduce workflows' memory to 68GiB

Previous bump to 70GiB pushed us up to the c5.12xlarge EC2 instances even though it was within the max of c5.9xlarge.¹ The max memory usage by the `generate_metadata` rule has been 64.76GiB over the last 4 days. Reducing memory for workflows to 68GiB to see if this would push us back down to the c5.9xlarge EC2 instances. ¹ https://bedfordlab.slack.com/archives/C01LCTT7JNN/p1674254476788549
nextstrain · Jan 24, 2023 · 714df3b · 714df3b
1 parent 6631c75
commit 714df3b
Show file tree

Hide file tree

Showing 8 changed files with 16 additions and 16 deletions.
diff --git a/.github/workflows/fetch-and-ingest-genbank-branch.yml b/.github/workflows/fetch-and-ingest-genbank-branch.yml
@@ -30,14 +30,14 @@ jobs:
           --no-download \
           --image nextstrain/ncov-ingest \
           --cpus 16 \
-          --memory 70GiB \
+          --memory 68GiB \
           --exec env \
           . \
             envdir env.d snakemake \
               --configfile config/genbank.yaml \
               --config "${config[@]}" \
               --cores 16 \
-              --resources mem_mb=70000 \
+              --resources mem_mb=68000 \
               --printshellcmds
       env:
         AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

diff --git a/.github/workflows/fetch-and-ingest-genbank-master.yml b/.github/workflows/fetch-and-ingest-genbank-master.yml
@@ -65,14 +65,14 @@ jobs:
           --no-download \
           --image nextstrain/ncov-ingest \
           --cpus 16 \
-          --memory 70GiB \
+          --memory 68GiB \
           --exec env \
           . \
             envdir env.d snakemake \
               --configfile config/genbank.yaml \
               --config "${config[@]}" \
               --cores 16 \
-              --resources mem_mb=70000 \
+              --resources mem_mb=68000 \
               --printshellcmds
       env:
         AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

diff --git a/.github/workflows/fetch-and-ingest-gisaid-branch.yml b/.github/workflows/fetch-and-ingest-gisaid-branch.yml
@@ -31,14 +31,14 @@ jobs:
           --no-download \
           --image nextstrain/ncov-ingest \
           --cpus 16 \
-          --memory 70GiB \
+          --memory 68GiB \
           --exec env \
           . \
             envdir env.d snakemake \
               --configfile config/gisaid.yaml \
               --config "${config[@]}" \
               --cores 16 \
-              --resources mem_mb=70000 \
+              --resources mem_mb=68000 \
               --printshellcmds
       env:
         AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

diff --git a/.github/workflows/fetch-and-ingest-gisaid-master.yml b/.github/workflows/fetch-and-ingest-gisaid-master.yml
@@ -66,14 +66,14 @@ jobs:
           --no-download \
           --image nextstrain/ncov-ingest \
           --cpus 16 \
-          --memory 70GiB \
+          --memory 68GiB \
           --exec env \
           . \
             envdir env.d snakemake \
               --configfile config/gisaid.yaml \
               --config "${config[@]}" \
               --cores 16 \
-              --resources mem_mb=70000 \
+              --resources mem_mb=68000 \
               --printshellcmds
       env:
         AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

diff --git a/.github/workflows/ingest-genbank-branch.yml b/.github/workflows/ingest-genbank-branch.yml
@@ -29,14 +29,14 @@ jobs:
             --no-download \
             --image nextstrain/ncov-ingest \
             --cpus 16 \
-            --memory 70GiB \
+            --memory 68GiB \
             --exec env \
             . \
               envdir env.d snakemake \
                 --configfile config/genbank.yaml \
                 --config "${config[@]}" \
                 --cores 16 \
-                --resources mem_mb=70000 \
+                --resources mem_mb=68000 \
                 --printshellcmds
         env:
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

diff --git a/.github/workflows/ingest-genbank-master.yml b/.github/workflows/ingest-genbank-master.yml
@@ -36,13 +36,13 @@ jobs:
           --no-download \
           --image nextstrain/ncov-ingest \
           --cpus 16 \
-          --memory 70GiB \
+          --memory 68GiB \
           --exec env \
           . \
             envdir env.d snakemake \
               --configfile config/genbank.yaml \
               --cores 16 \
-              --resources mem_mb=70000 \
+              --resources mem_mb=68000 \
               --printshellcmds
       env:
         AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

diff --git a/.github/workflows/ingest-gisaid-branch.yml b/.github/workflows/ingest-gisaid-branch.yml
@@ -28,14 +28,14 @@ jobs:
             --no-download \
             --image nextstrain/ncov-ingest \
             --cpus 16 \
-            --memory 70GiB \
+            --memory 68GiB \
             --exec env \
             . \
               envdir env.d snakemake \
                 --configfile config/gisaid.yaml \
                 --config "${config[@]}" \
                 --cores 16 \
-                --resources mem_mb=70000 \
+                --resources mem_mb=68000 \
                 --printshellcmds
         env:
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

diff --git a/.github/workflows/ingest-gisaid-master.yml b/.github/workflows/ingest-gisaid-master.yml
@@ -36,13 +36,13 @@ jobs:
           --no-download \
           --image nextstrain/ncov-ingest \
           --cpus 16 \
-          --memory 70GiB \
+          --memory 68GiB \
           --exec env \
           . \
             envdir env.d snakemake \
               --configfile config/gisaid.yaml \
               --cores 16 \
-              --resources mem_mb=70000 \
+              --resources mem_mb=68000 \
               --printshellcmds
       env:
         AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}