-
Notifications
You must be signed in to change notification settings - Fork 2
/
jobcard_submission.py
51 lines (41 loc) · 1.53 KB
/
jobcard_submission.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
'''
Description: a utility for creating and submitting job cards to run gen_training_X.Xd.py scripts in parallel.
Author: Sadegh Sadeghi Tabas (sadegh.tabas@noaa.gov)
Revision history:
-20231106: Sadegh Tabas, initial code
'''
import os
from datetime import datetime, timedelta
# Define the start and end dates
start_date = datetime(2021, 3, 23)
end_date = datetime(2023, 10, 31)
delta = timedelta(days=5) # Interval of 10 days
process = 'gfs'
resolution = '1.0'
# Template of the job card
job_card_template = """#!/bin/bash
#SBATCH -J {process}_process
#SBATCH -o {process}_output_{start}_{end}.out
#SBATCH -e {process}_err_{start}_{end}.err
#SBATCH -N 1
#SBATCH -t 30-00:00:00 # 1 month (30 days)
# Load Miniconda module
source ~/.bashrc
# Activate the Conda environment
conda activate ml4bc
# Navigate to the working directory
cd /lustre/ML4BC
# Run the Python script
python3 gen_training_{resolution}d.py {start} {end} -p {process} -o /contrib/$USER/ML4BC/
"""
# Generate and submit job cards
while start_date <= end_date:
next_date = start_date + delta
job_card = job_card_template.format(process=process, start=start_date.strftime("%Y%m%d"), end=next_date.strftime("%Y%m%d"), resolution=resolution)
# Write the job card to a file
job_filename = f"{process}_job_{start_date.strftime('%Y%m%d')}_{next_date.strftime('%Y%m%d')}.sh"
with open(job_filename, 'w') as job_file:
job_file.write(job_card)
# Submit the job
os.system(f"sbatch {job_filename}")
start_date = next_date + timedelta(days=1)