Generate working volume in run-fv3gfs workflow template

ai2cm · Aug 25, 2020 · 2c0f223 · 2c0f223
1 parent ff53a96
commit 2c0f223
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 14 deletions.
diff --git a/tests/end_to_end_integration/argo.yaml b/tests/end_to_end_integration/argo.yaml
@@ -9,14 +9,6 @@ spec:
       key: dedicated
       value: climate-sim-pool
   entrypoint: main
-  volumeClaimTemplates:
-    - metadata:
-        name: workdir
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 5Gi
   volumes:
     - name: restart-data
       persistentVolumeClaim:
@@ -45,6 +37,7 @@ spec:
             value: "{{workflow.parameters.root}}/nudging"
           - name: nudging-config
             value: "{{workflow.parameters.nudging-config}}"
+          - {name: workdir-size, value: 5Gi}  # use a small workdir volume for e2e test
       - name: train-model
         dependencies: [nudging-run]
         templateRef:

diff --git a/workflows/argo/nudging/nudging.yaml b/workflows/argo/nudging/nudging.yaml
@@ -18,8 +18,12 @@ spec:
       - {name: submission-count, value: "1"}
       - {name: cpu, value: "6"}
       - {name: memory, value: 8Gi}
+      - {name: workdir-size, value: 100Gi}
   entrypoint: main
   volumes:
+    - name: restart-data
+      persistentVolumeClaim:
+        claimName: nudging-read-only
     - name: gcp-key-secret
       secret:
         defaultMode: 420
@@ -46,6 +50,7 @@ spec:
               - {name: submission-count, value: "{{workflow.parameters.submission-count}}"}
               - {name: cpu, value: "{{workflow.parameters.cpu}}"}
               - {name: memory, value: "{{workflow.parameters.memory}}"}
+              - {name: workdir-size, value: "{{workflow.parameters.workdir-size}}"}
     - name: nudging
       inputs:
         parameters:
@@ -60,6 +65,7 @@ spec:
           - {name: submission-count, value: "1"}
           - {name: cpu, value: "6"}
           - {name: memory, value: 8Gi}
+          - {name: workdir-size, value: 100Gi}
       steps:
         - - template: prepare-config
             name: prepare-config

diff --git a/workflows/argo/run-fv3gfs.yaml b/workflows/argo/run-fv3gfs.yaml
@@ -14,6 +14,7 @@ spec:
       - {name: cpu, value: "6"}
       - {name: memory, value: 4Gi}
       - {name: submission-count, value: "1"}
+      - {name: workdir-size, value: 100Gi}
   entrypoint: run-fv3gfs
   volumes:
     - name: gcp-key-secret
@@ -33,14 +34,21 @@ spec:
         - {name: cpu, value: "6"}
         - {name: memory, value: 8Gi}
         - {name: submission-count, value: "1"}
+        - {name: workdir-size, value: 100Gi}
     steps:
+      - - name: generate-volume
+          template: generate-volume
+          arguments:
+            parameters:
+            - {name: workdir-size, value: "{{inputs.parameters.workdir-size}}"}
       - - name: write-configs
           template: write-configs
           arguments:
             parameters:
             - {name: fv3config, value: "{{inputs.parameters.fv3config}}"}
             - {name: runfile, value: "{{inputs.parameters.runfile}}"}
             - {name: chunks, value: "{{inputs.parameters.chunks}}"}
+            - {name: workdir-name, value: "{{steps.generate-volume.outputs.parameters.workdir-name}}"}
       - - name: run-first-segment
           template: run-all-segments
           arguments:
@@ -52,16 +60,18 @@ spec:
             - {name: memory, value: "{{inputs.parameters.memory}}"}
             - {name: submission-count, value: "{{inputs.parameters.submission-count}}"}
             - {name: step, value: 0}
+            - {name: workdir-name, value: "{{steps.generate-volume.outputs.parameters.workdir-name}}"}
   - name: write-configs
     inputs:
       parameters:
         - name: fv3config
         - name: runfile
         - name: chunks
+        - name: workdir-name
     container:
       image: ubuntu:latest
       volumeMounts:
-        - name: workdir
+        - name: "{{inputs.parameters.workdir-name}}"
           mountPath: /mnt/data
       command: ["/bin/bash", "-c", "-x"]
       args:
@@ -82,6 +92,7 @@ spec:
         - name: memory
         - name: submission-count
         - name: step
+        - name: workdir-name
     steps:
       - - template: run-segment
           name: run-segment
@@ -92,14 +103,15 @@ spec:
             - {name: output-url, value: "{{inputs.parameters.output-url}}"}
             - {name: cpu, value: "{{inputs.parameters.cpu}}"}
             - {name: memory, value: "{{inputs.parameters.memory}}"}
+            - {name: workdir-name, value: "{{inputs.parameters.workdir-name}}"}
       - - name: increment-step
           template: increment-step
           arguments:
             parameters:
               - {name: step, value: "{{inputs.parameters.step}}"}
       - - name: run-next-segment
           template: run-all-segments
-          when: "{{inputs.parameters.step}} < {{inputs.parameters.submission-count}} - 1"
+          when: "{{steps.increment-step.outputs.result}} < {{inputs.parameters.submission-count}}"
           arguments:
             parameters:
             - {name: fv3gfs-image, value: "{{inputs.parameters.fv3gfs-image}}"}
@@ -109,6 +121,7 @@ spec:
             - {name: memory, value: "{{inputs.parameters.memory}}"}
             - {name: submission-count, value: "{{inputs.parameters.submission-count}}"}
             - {name: step, value: "{{steps.increment-step.outputs.result}}"}
+            - {name: workdir-name, value: "{{inputs.parameters.workdir-name}}"}
   - name: run-segment
     inputs:
       parameters:
@@ -117,6 +130,7 @@ spec:
         - name: output-url
         - name: cpu
         - name: memory
+        - name: workdir-name
     steps:
       - - name: run-model
           template: run-model
@@ -125,17 +139,20 @@ spec:
               - {name: fv3gfs-image, value: "{{inputs.parameters.fv3gfs-image}}"}
               - {name: cpu, value: "{{inputs.parameters.cpu}}"}
               - {name: memory, value: "{{inputs.parameters.memory}}"}
+              - {name: workdir-name, value: "{{inputs.parameters.workdir-name}}"}
       - - name: upload-outputs
           template: post-process
           arguments:
             parameters:
               - {name: post-process-image, value: "{{inputs.parameters.post-process-image}}"}
               - {name: output-url, value: "{{inputs.parameters.output-url}}"}
+              - {name: workdir-name, value: "{{inputs.parameters.workdir-name}}"}
       - - name: update-config
           template: update-config
           arguments:
             parameters:
               - {name: restarts-url, value: "/mnt/data/RESTART"}
+              - {name: workdir-name, value: "{{inputs.parameters.workdir-name}}"}
   - name: run-model
     inputs:
       parameters:
@@ -174,7 +191,7 @@ spec:
       terminationMessagePath: /dev/termination-log
       terminationMessagePolicy: File
       volumeMounts:
-        - name: workdir
+        - name: "{{inputs.parameters.workdir-name}}"
           mountPath: /mnt/data
         - name: restart-data
           mountPath: /mnt/input
@@ -202,7 +219,7 @@ spec:
         rm -r /mnt/data/rundir
         rm -r /tmp/processed_rundir
       volumeMounts:
-        - name: workdir
+        - name: "{{inputs.parameters.workdir-name}}"
           mountPath: /mnt/data
         - name: gcp-key-secret
           mountPath: /secret/gcp-credentials
@@ -231,7 +248,7 @@ spec:
         yq w -i /mnt/data/fv3config.yaml namelist.fv_core_nml.na_init 0
         yq w -i /mnt/data/fv3config.yaml namelist.coupler_nml.force_date_from_namelist false
       volumeMounts:
-        - name: workdir
+        - name: "{{inputs.parameters.workdir-name}}"
           mountPath: /mnt/data
   - name: increment-step
     inputs:
@@ -241,4 +258,26 @@ spec:
       image: python:alpine3.6
       command: [python]
       source: |
-        print({{inputs.parameters.step}} + 1)
+        print({{inputs.parameters.step}} + 1)
+  - name: generate-volume
+    inputs:
+      parameters:
+        - name: workdir-size
+    resource:
+      action: create
+      setOwnerReference: true
+      manifest: |
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          generateName: workdir-
+        spec:
+          accessModes: ['ReadWriteOnce']
+          resources:
+            requests:
+              storage: "{{inputs.parameters.workdir-size}}"
+    outputs:
+      parameters:
+        - name: workdir-name
+          valueFrom:
+            jsonPath: "{.metadata.name}"