Devoto dev #258
Devoto dev #258
Conversation
|
I put it in draft to prevent accidental merges. In the meanwhile I will start checking it then we can ask Julian and Alessio for a review :) |
| # ST0320 ongoing -> 'service' end date | ||
|
|
||
| "Loops over all runs of all nights" | ||
| Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/*")) |
There was a problem hiding this comment.
Here we must be careful about the current structure of the directories
| for night in Nights_list: | ||
| "Night period" | ||
| night_date=night.split('/')[-1] | ||
| print('night',night_date) |
There was a problem hiding this comment.
Of course, when we manage to make the scripts work fine, we have to remove some of the 'useless' print
| ): | ||
| period = ST_list[p] | ||
|
|
||
| Run_list = glob.glob(f"{night}/Merged/*.h5") |
There was a problem hiding this comment.
again, directories
| nsb = nsb_list[j] | ||
| "Writing on output .txt file" | ||
| if (nsb <= 3.1): | ||
| with open(f"{night}/Merged/logs/{period}_{nsb}.txt","a+") as file: |
There was a problem hiding this comment.
again, directories
| """ | ||
|
|
||
| process_name = source | ||
| Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/*")) |
There was a problem hiding this comment.
again, directories
| RFdir=f'/fefs/aswg/LST1MAGIC/mc/models/{p}/NSB{nsb}/v01.2/dec_2276/' | ||
| with open(f'{source}_DL1_to_DL2_{night_date}_{file.split("/")[-1].rstrip("txt")}sh', "w") as f: | ||
| f.write("#!/bin/sh\n\n") | ||
| f.write("#SBATCH -p long\n") |
There was a problem hiding this comment.
maybe here we can switch to short (and check memory usage and job accounting) to prevent issues with IT
| f.write("#!/bin/sh\n\n") | ||
| f.write("#SBATCH -p long\n") | ||
| f.write("#SBATCH -J " + process_name + "\n") | ||
| f.write(f"#SBATCH --array=0-{process_size}%100\n") |
There was a problem hiding this comment.
here we can remove limit on job array
| f.write("#SBATCH -J " + process_name + "\n") | ||
| f.write(f"#SBATCH --array=0-{process_size}%100\n") | ||
| f.write("#SBATCH --mem=90g\n") | ||
| f.write("#SBATCH -N 1\n\n") |
| p=file.split("/")[-1].split("_")[0] | ||
| RFdir=f'/fefs/aswg/LST1MAGIC/mc/models/{p}/NSB{nsb}/v01.2/dec_2276/' | ||
| with open(f'{source}_DL1_to_DL2_{night_date}_{file.split("/")[-1].rstrip("txt")}sh', "w") as f: | ||
| f.write("#!/bin/sh\n\n") |
There was a problem hiding this comment.
move to slurm_lines function so that also log-files management would be easier
| list_of_stereo_scripts = np.sort(glob.glob(f'{source_name}_DL1_to_DL2*.sh')) | ||
| print(list_of_stereo_scripts) | ||
| for n, run in enumerate(list_of_stereo_scripts): | ||
| if n == 0: |
There was a problem hiding this comment.
here we can improve these lines as in the previous scripts
| f = open(str(target_dir) + "/config_DL3.yaml", "w") | ||
| f.write( | ||
| "mc_tel_ids:\n LST-1: " | ||
| + str(ids[0]) |
There was a problem hiding this comment.
here we can fix it (e.g., see previous scripts)
|
|
||
| target_dir = str(target_dir) | ||
| if not os.path.exists(target_dir + f"/v{__version__}/{source}/DL3"): | ||
| os.mkdir(target_dir + f"/v{__version__}/{source}/DL3") |
| period = file.split("/")[-1].split("_")[0] | ||
| print("period = ", period) | ||
|
|
||
| IRF_dir = ( |
There was a problem hiding this comment.
As for RFs, we will have to pass a general file and read mc dec, wobble...
| output = target_dir + f"/v{__version__}/{source}/DL3" | ||
|
|
||
| f = open(f'{source}_DL3_{nsb}_{period}_{night.split("/")[-1]}.sh', "w") | ||
| f.write("#!/bin/sh\n\n") |
| telescope_ids = list(config["mc_tel_ids"].values()) | ||
| target_dir = Path(config["directories"]["workspace_dir"]) | ||
|
|
||
| target_coords = [ |
There was a problem hiding this comment.
read from LST database for each source
| cluster = config["general"]["cluster"] | ||
|
|
||
| #cp the .txt files from DL1 stereo anaysis to be used again. | ||
| DL1stereo_Nihts = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/*")) |
| list_of_stereo_scripts = np.sort(glob.glob(f'{source_name}_DL3*.sh')) | ||
| print(list_of_stereo_scripts) | ||
| for n, run in enumerate(list_of_stereo_scripts): | ||
| if n == 0: |
There was a problem hiding this comment.
as above, can be improved
Adding two semiautomatic scripts to create DL2 and DL3 files, currently the scripts are not working properly, I'm working on a solution with Elisa.