From fcf237e818fd13d08f745c9fdaa045c7162d069f Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 10 Sep 2024 10:57:23 +0200 Subject: [PATCH 1/9] initial test --- mkdocs/extra/gent.yml | 1 + scripts/available_software/available_software.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mkdocs/extra/gent.yml b/mkdocs/extra/gent.yml index 20179bd6fce..f7daade1ffd 100644 --- a/mkdocs/extra/gent.yml +++ b/mkdocs/extra/gent.yml @@ -16,6 +16,7 @@ hpcname: hpcugent loginnode: login.hpc.ugent.be loginhost: gligar07.gastly.os altloginhost: gligar08.gastly.os +modules_last_updated: 2021-09-01 # get these with ssh-keyscan gligar01.ugent.be > file # ssh-keygen -l -f file # ssh-keygen -l -f file -E md5 diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py index 5c51e4995f9..f9fab56dfa6 100644 --- a/scripts/available_software/available_software.py +++ b/scripts/available_software/available_software.py @@ -381,7 +381,7 @@ def generate_software_detail_page( md_file.new_paragraph(f"To start using {software_name}, load one of these modules using a `module load` command " f"like:") md_file.insert_code(f"module load {newest_version}", language="shell") - md_file.new_paragraph(f"(This data was automatically generated on {generated_time})", bold_italics_code="i") + md_file.new_paragraph("(This data was automatically generated on {{modules_last_updated}})", bold_italics_code="i") md_file.new_line() md_file.new_table( From 3e466d7b7551e2843febeb28531a955b12fa3355 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 10 Sep 2024 11:32:57 +0200 Subject: [PATCH 2/9] script changes yaml containing generated time everytime it runs --- .../available_software/available_software.py | 32 ++++++++++++++++--- scripts/available_software/requirements.txt | 3 +- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py index f9fab56dfa6..a60e2761f31 100644 --- a/scripts/available_software/available_software.py +++ b/scripts/available_software/available_software.py @@ -39,6 +39,7 @@ import numpy as np from mdutils.mdutils import MdUtils from natsort import natsorted +import yaml # -------------------------------------------------------------------------------------------------------- @@ -77,7 +78,9 @@ def main(): json_path = generate_json_detailed(modules, path_data_dir) print("Done!") print("Generate detailed pages... ", end="", flush=True) - generate_detail_pages(json_path, os.path.join(root_dir, "mkdocs/docs/HPC/only/gent/available_software/detail")) + detail_folder = os.path.join(root_dir, "mkdocs/docs/HPC/only/gent/available_software/detail") + generated_time_yml = os.path.join(root_dir, "mkdocs/extra/gent.yml") # yml containing the time the data was generated + generate_detail_pages(json_path, detail_folder, generated_time_yml) print("Done!") @@ -356,7 +359,6 @@ def generate_software_table_data(software_data: dict, clusters: list) -> list: def generate_software_detail_page( software_name: str, software_data: dict, - generated_time: str, clusters: list, path: str ) -> None: @@ -365,7 +367,6 @@ def generate_software_detail_page( @param software_name: Name of the software @param software_data: Additional information about the software (version, etc...) - @param generated_time: Timestamp when the data was generated @param clusters: List with all the cluster names @param path: Path of the directory where the detailed page will be created. """ @@ -399,7 +400,7 @@ def generate_software_detail_page( f.write("---\nhide:\n - toc\n---\n" + read_data) -def generate_detail_pages(json_path, dest_path) -> None: +def generate_detail_pages(json_path, dest_path, generated_time_yml) -> None: """ Generate all the detailed pages for all the software that is available. """ @@ -407,9 +408,30 @@ def generate_detail_pages(json_path, dest_path) -> None: with open(json_path) as json_data: data = json.load(json_data) + # update the time the data was generated + update_generated_time_yml(generated_time_yml, data["time_generated"]) + all_clusters = data["clusters"] for software, content in data["software"].items(): - generate_software_detail_page(software, content, data["time_generated"], all_clusters, dest_path) + generate_software_detail_page(software, content, all_clusters, dest_path) + + +def update_generated_time_yml(generated_time_yml, generated_time) -> None: + """ + Update the time the data was generated in the YAML file. + + @param generated_time_yml: Path to the YAML file containing the field 'modules_last_updated' + @param generated_time: Time the data was generated + """ + with open(generated_time_yml, 'r') as file: + data = yaml.safe_load(file) + + # Update the 'modules_last_updated' field + data['modules_last_updated'] = generated_time # Set the new date here + + # Write the updated YAML back to the file + with open(generated_time_yml, 'w') as file: + yaml.safe_dump(data, file) # -------------------------------------------------------------------------------------------------------- diff --git a/scripts/available_software/requirements.txt b/scripts/available_software/requirements.txt index 1b7478cc435..84e1b270ed4 100644 --- a/scripts/available_software/requirements.txt +++ b/scripts/available_software/requirements.txt @@ -1,3 +1,4 @@ mdutils numpy -natsort \ No newline at end of file +natsort +PyYAML \ No newline at end of file From d3c2907dce3315cd514a73c39e72bc1a4eb4be6b Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 10 Sep 2024 12:02:42 +0200 Subject: [PATCH 3/9] dont use yaml --- .../available_software/available_software.py | 18 +++++++++--------- scripts/available_software/requirements.txt | 3 +-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py index a60e2761f31..43c945909c8 100644 --- a/scripts/available_software/available_software.py +++ b/scripts/available_software/available_software.py @@ -39,7 +39,7 @@ import numpy as np from mdutils.mdutils import MdUtils from natsort import natsorted -import yaml +from ruamel.yaml import YAML # -------------------------------------------------------------------------------------------------------- @@ -423,17 +423,17 @@ def update_generated_time_yml(generated_time_yml, generated_time) -> None: @param generated_time_yml: Path to the YAML file containing the field 'modules_last_updated' @param generated_time: Time the data was generated """ - with open(generated_time_yml, 'r') as file: - data = yaml.safe_load(file) + key = "modules_last_updated" - # Update the 'modules_last_updated' field - data['modules_last_updated'] = generated_time # Set the new date here + # Read the file and replace the specific line + with open(generated_time_yml, 'r') as file: + lines = file.readlines() - # Write the updated YAML back to the file with open(generated_time_yml, 'w') as file: - yaml.safe_dump(data, file) - - + for line in lines: + if line.startswith(key): + line = f"{key}: {generated_time}\n" + file.write(line) # -------------------------------------------------------------------------------------------------------- # Generate overview markdown # -------------------------------------------------------------------------------------------------------- diff --git a/scripts/available_software/requirements.txt b/scripts/available_software/requirements.txt index 84e1b270ed4..1b7478cc435 100644 --- a/scripts/available_software/requirements.txt +++ b/scripts/available_software/requirements.txt @@ -1,4 +1,3 @@ mdutils numpy -natsort -PyYAML \ No newline at end of file +natsort \ No newline at end of file From dea78706a862d004d9d34a41a9f6a0fdb2174236 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 10 Sep 2024 12:09:19 +0200 Subject: [PATCH 4/9] remove ruamel import --- scripts/available_software/available_software.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py index 43c945909c8..6477e5d890d 100644 --- a/scripts/available_software/available_software.py +++ b/scripts/available_software/available_software.py @@ -39,7 +39,6 @@ import numpy as np from mdutils.mdutils import MdUtils from natsort import natsorted -from ruamel.yaml import YAML # -------------------------------------------------------------------------------------------------------- From 8906cb69f48f5a239fad04519edcdec295a58c02 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 10 Sep 2024 12:14:14 +0200 Subject: [PATCH 5/9] Add warning when line could not be replaced --- scripts/available_software/available_software.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py index 6477e5d890d..03e04d0dca1 100644 --- a/scripts/available_software/available_software.py +++ b/scripts/available_software/available_software.py @@ -428,11 +428,16 @@ def update_generated_time_yml(generated_time_yml, generated_time) -> None: with open(generated_time_yml, 'r') as file: lines = file.readlines() + replaced = False with open(generated_time_yml, 'w') as file: for line in lines: if line.startswith(key): line = f"{key}: {generated_time}\n" + replaced = True file.write(line) + + if not replaced: + print(f"WARNING: Could not find the key '{key}' in the YAML file '{generated_time_yml}'") # -------------------------------------------------------------------------------------------------------- # Generate overview markdown # -------------------------------------------------------------------------------------------------------- From c2e965c22b4ba7d9ad1f4cbf4d6492c6b5b10af4 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 10 Sep 2024 12:25:19 +0200 Subject: [PATCH 6/9] add extra description to docstring --- scripts/available_software/available_software.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py index 03e04d0dca1..862bfe5caf0 100644 --- a/scripts/available_software/available_software.py +++ b/scripts/available_software/available_software.py @@ -418,6 +418,7 @@ def generate_detail_pages(json_path, dest_path, generated_time_yml) -> None: def update_generated_time_yml(generated_time_yml, generated_time) -> None: """ Update the time the data was generated in the YAML file. + This is done by updating the field 'modules_last_updated'. @param generated_time_yml: Path to the YAML file containing the field 'modules_last_updated' @param generated_time: Time the data was generated From 23d95cc063c70b5bad638a50038cae6591243d4e Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 10 Sep 2024 12:25:39 +0200 Subject: [PATCH 7/9] replace dummy with actual last update --- mkdocs/extra/gent.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/extra/gent.yml b/mkdocs/extra/gent.yml index f7daade1ffd..098a1811738 100644 --- a/mkdocs/extra/gent.yml +++ b/mkdocs/extra/gent.yml @@ -16,7 +16,7 @@ hpcname: hpcugent loginnode: login.hpc.ugent.be loginhost: gligar07.gastly.os altloginhost: gligar08.gastly.os -modules_last_updated: 2021-09-01 +modules_last_updated: Mon, 09 Sep 2024 at 14:06:35 CEST # get these with ssh-keyscan gligar01.ugent.be > file # ssh-keygen -l -f file # ssh-keygen -l -f file -E md5 From 32d04388ca72a45490ce24b74a5e1cb1a61a1f52 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 10 Sep 2024 12:39:52 +0200 Subject: [PATCH 8/9] add comment to end of line --- mkdocs/extra/gent.yml | 2 +- scripts/available_software/available_software.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mkdocs/extra/gent.yml b/mkdocs/extra/gent.yml index 098a1811738..7ec51762329 100644 --- a/mkdocs/extra/gent.yml +++ b/mkdocs/extra/gent.yml @@ -16,7 +16,7 @@ hpcname: hpcugent loginnode: login.hpc.ugent.be loginhost: gligar07.gastly.os altloginhost: gligar08.gastly.os -modules_last_updated: Mon, 09 Sep 2024 at 14:06:35 CEST +modules_last_updated: Mon, 09 Sep 2024 at 14:06:35 CEST # This line is automatically updated by scripts/available_modules/available_modules.py # get these with ssh-keyscan gligar01.ugent.be > file # ssh-keygen -l -f file # ssh-keygen -l -f file -E md5 diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py index 862bfe5caf0..828e060512b 100644 --- a/scripts/available_software/available_software.py +++ b/scripts/available_software/available_software.py @@ -433,12 +433,14 @@ def update_generated_time_yml(generated_time_yml, generated_time) -> None: with open(generated_time_yml, 'w') as file: for line in lines: if line.startswith(key): - line = f"{key}: {generated_time}\n" + line = f"{key}: {generated_time} # This line is automatically updated by scripts/available_modules/available_modules.py\n" replaced = True file.write(line) if not replaced: print(f"WARNING: Could not find the key '{key}' in the YAML file '{generated_time_yml}'") + + # -------------------------------------------------------------------------------------------------------- # Generate overview markdown # -------------------------------------------------------------------------------------------------------- From 6b21da5eee73dff1209585ceaa508bdf32994807 Mon Sep 17 00:00:00 2001 From: Lukas Barragan Torres Date: Tue, 10 Sep 2024 13:10:28 +0200 Subject: [PATCH 9/9] linting --- scripts/available_software/available_software.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py index 828e060512b..5af8778f771 100644 --- a/scripts/available_software/available_software.py +++ b/scripts/available_software/available_software.py @@ -78,7 +78,7 @@ def main(): print("Done!") print("Generate detailed pages... ", end="", flush=True) detail_folder = os.path.join(root_dir, "mkdocs/docs/HPC/only/gent/available_software/detail") - generated_time_yml = os.path.join(root_dir, "mkdocs/extra/gent.yml") # yml containing the time the data was generated + generated_time_yml = os.path.join(root_dir, "mkdocs/extra/gent.yml") # yml containing time the data was generated generate_detail_pages(json_path, detail_folder, generated_time_yml) print("Done!") @@ -433,7 +433,8 @@ def update_generated_time_yml(generated_time_yml, generated_time) -> None: with open(generated_time_yml, 'w') as file: for line in lines: if line.startswith(key): - line = f"{key}: {generated_time} # This line is automatically updated by scripts/available_modules/available_modules.py\n" + comment = "# This line is automatically updated by scripts/available_modules/available_modules.py" + line = f"{key}: {generated_time} {comment}\n" replaced = True file.write(line)