Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add the Cylc language. #6832

Merged
merged 6 commits into from
Aug 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
[submodule "vendor/grammars/ColdFusion"]
path = vendor/grammars/ColdFusion
url = https://github.com/SublimeText/ColdFusion
[submodule "vendor/grammars/Cylc.tmbundle"]
path = vendor/grammars/Cylc.tmbundle
url = https://github.com/cylc/Cylc.tmbundle.git
[submodule "vendor/grammars/Dafny-VSCode"]
path = vendor/grammars/Dafny-VSCode
url = https://github.com/DafnyVSCode/Dafny-VSCode
Expand Down
2 changes: 2 additions & 0 deletions grammars.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ vendor/grammars/ColdFusion:
- source.cfscript.cfc
- text.cfml.basic
- text.html.cfm
vendor/grammars/Cylc.tmbundle:
- source.cylc
vendor/grammars/Dafny-VSCode:
- text.dfy.dafny
vendor/grammars/Docker.tmbundle:
Expand Down
11 changes: 11 additions & 0 deletions lib/linguist/languages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1399,6 +1399,17 @@ Cycript:
codemirror_mode: javascript
codemirror_mime_type: text/javascript
language_id: 78
Cylc:
type: data
color: "#00b3fd"
extensions:
- ".cylc"
filenames:
- suite.rc
tm_scope: source.cylc
ace_mode: ini
group: INI
language_id: 476447814
Cypher:
type: programming
color: "#34c0eb"
Expand Down
82 changes: 82 additions & 0 deletions samples/Cylc/cset-include.cylc
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# Modified code samples from https://github.com/MetOffice/CSET/blob/main/cset-workflow/includes
# (Apache 2.0 license)

{% if DOMAIN_MEAN_SURFACE_TIME_SERIES %}
{% for model_field in SURFACE_MODEL_FIELDS %}
[runtime]
[[pre_process_domain_mean_surface_time_series_{{model_field}}]]
inherit = PARALLEL
[[[environment]]]
CSET_RECIPE_NAME = "generic_surface_domain_mean_time_series.yaml"
CSET_ADDOPTS = "--VARNAME={{model_field}}"

[[collate_domain_mean_surface_time_series_{{model_field}}]]
inherit = COLLATE
[[[environment]]]
CSET_RECIPE_NAME = "generic_surface_domain_mean_time_series.yaml"
CSET_ADDOPTS = "--VARNAME={{model_field}}"
{% endfor %}
{% endif %}

{% if DETERMINISTIC_PLOT_CAPE_RATIO %}
[runtime]
[[parallel_plot_cape_ratio]]
inherit = PARALLEL
[[[environment]]]
CSET_RECIPE_NAME = "CAPE_ratio_plot.yaml"

[[collate_plot_cape_ratio]]
inherit = COLLATE
[[[environment]]]
CSET_RECIPE_NAME = "CAPE_ratio_plot.yaml"
{% endif %}

{% if DOMAIN_MEAN_TIME_SERIES_STASH %}
{% for stash in STASH_CODES %}
[runtime]
[[pre_process_stash_surface_domain_mean_time_series_{{stash}}]]
inherit = PARALLEL
[[[environment]]]
CSET_RECIPE_NAME = "stash_surface_domain_mean_time_series.yaml"
CSET_ADDOPTS = "--STASH={{stash}}"

[[collate_stash_surface_domain_mean_time_series_{{stash}}]]
inherit = COLLATE
[[[environment]]]
CSET_RECIPE_NAME = "stash_surface_domain_mean_time_series.yaml"
CSET_ADDOPTS = "--STASH={{stash}}"
{% endfor %}
{% endif %}

{% if LFRIC_DOMAIN_MEAN_SURFACE_TIME_SERIES %}
{% for model_field in SURFACE_MODEL_FIELDS %}
[runtime]
[[pre_process_lfric_domain_mean_surface_time_series_{{model_field}}]]
inherit = PARALLEL
[[[environment]]]
CSET_RECIPE_NAME = "lfric_generic_surface_domain_mean_time_series.yaml"
CSET_ADDOPTS = """
--VARNAME={{model_field}}
{% if SELECT_SUBAREA %}
--SUBAREA_LAT_BOUND_BOTTOM={{SUBAREA_LAT_BOUND_BOTTOM}}
--SUBAREA_LAT_BOUND_TOP={{SUBAREA_LAT_BOUND_TOP}}
--SUBAREA_LON_BOUND_RIGHT={{SUBAREA_LON_BOUND_RIGHT}}
--SUBAREA_LON_BOUND_LEFT={{SUBAREA_LON_BOUND_LEFT}}
{% endif %}
"""

[[collate_lfric_domain_mean_surface_time_series_{{model_field}}]]
inherit = COLLATE
[[[environment]]]
CSET_RECIPE_NAME = "lfric_generic_surface_domain_mean_time_series.yaml"
CSET_ADDOPTS = """
--VARNAME={{model_field}}
{% if SELECT_SUBAREA %}
--SUBAREA_LAT_BOUND_BOTTOM={{SUBAREA_LAT_BOUND_BOTTOM}}
--SUBAREA_LAT_BOUND_TOP={{SUBAREA_LAT_BOUND_TOP}}
--SUBAREA_LON_BOUND_RIGHT={{SUBAREA_LON_BOUND_RIGHT}}
--SUBAREA_LON_BOUND_LEFT={{SUBAREA_LON_BOUND_LEFT}}
{% endif %}
"""
{% endfor %}
{% endif %}
159 changes: 159 additions & 0 deletions samples/Cylc/cset.cylc
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
# Modified code sample from https://github.com/MetOffice/CSET/blob/main/cset-workflow/flow.cylc
# (Apache 2.0 license)

[meta]
title = CSET
description = Workflow for running CSET.
URL = https://metoffice.github.io/CSET

[scheduler]
UTC mode = True

[scheduling]
runahead limit = P{{CSET_RUNAHEAD_LIMIT}}
initial cycle point = {{CSET_INITIAL_CYCLE_POINT}}
final cycle point = {{CSET_FINAL_CYCLE_POINT}}

[[graph]]
# Only runs on the first cycle.
R1/^ = """
build_conda => install_website_skeleton
build_conda => install_local_cset
install_website_skeleton & install_local_cset => FETCH_DATA
"""

# Only runs on the final cycle.
R1/$ = """
process_finish => COLLATE:succeed-all =>
finish_website => send_email => housekeeping_full
"""

# Runs every cycle to process the data in parallel.
{{CSET_CYCLE_PERIOD}} = """
install_website_skeleton[^] & install_local_cset[^] =>
FETCH_DATA:succeed-all => PARALLEL:succeed-all =>
process_finish => housekeeping_raw

# Intercycle dependence with this task ensures the collate step waits for
# the required data.
process_finish[-{{CSET_CYCLE_PERIOD}}] => process_finish
"""

{% if CSET_INCREMENTAL_OUTPUT %}
# Runs every so often to update output plots during runtime.
{{CSET_INCREMENTAL_OUTPUT_PERIOD}} = """
COLLATE[-{{CSET_INCREMENTAL_OUTPUT_PERIOD}}]:finish-all &
process_finish => COLLATE
"""
{% endif %}

[runtime]
[[root]]
script = rose task-run -v
execution time limit = PT15M
[[[environment]]]
CSET_ENV_USE_MODULES = {{CSET_ENV_USE_MODULES}}
{% if CSET_ENV_USE_MODULES %}
MODULES_LIST = {{MODULES_LIST}}
MODULES_PURGE = {{MODULES_PURGE}}
{% endif %}

CSET_ENV_USE_CONDA = {{CSET_ENV_USE_CONDA}}
{% if CSET_ENV_USE_CONDA %}
CONDA_PATH = {{CONDA_PATH}}
CONDA_VENV_LOCATION = {{CONDA_VENV_LOCATION}}
{% endif %}

CSET_ENV_SEPARATE_MET = {{CSET_ENV_SEPARATE_MET}}
{% if CSET_ENV_SEPARATE_MET %}
CONDA_METPLUS_VENV_LOCATION = {{CONDA_METPLUS_VENV_LOCATION}}
MET_INSTALL_DIR = {{MET_INSTALL_DIR}}
MET_BUILD_BASE = {{MET_BUILD_BASE}}
METPLUS_BASE = {{METPLUS_BASE}}
MET_LIBRARIES = {{MET_LIBRARIES}}
{% endif %}

LOGLEVEL = {{LOGLEVEL}}
WEB_DIR = {{WEB_DIR}}
COLORBAR_FILE = {{COLORBAR_FILE}}

[[PARALLEL]]
script = rose task-run -v --app-key=run_cset_recipe
[[[environment]]]
CSET_BAKE_MODE = parallel

[[COLLATE]]
script = rose task-run -v --app-key=run_cset_recipe
[[[environment]]]
CSET_BAKE_MODE = collate

[[FETCH_DATA]]

[[process_finish]]
# Dummy task needed for workflow scheduling.
script = true
platform = localhost

[[build_conda]]
# Create the conda environment if it does not yet exist.
execution time limit = PT30M
[[[environment]]]
CONDA_VENV_CREATE = {{CONDA_VENV_CREATE}}

[[install_local_cset]]
# Install CSET from source.
execution time limit = PT5M
[[[environment]]]
CSET_ENV_USE_LOCAL_CSET = {{CSET_ENV_USE_LOCAL_CSET}}
{% if CSET_ENV_USE_LOCAL_CSET %}
CSET_LOCAL_CSET_PATH = {{CSET_LOCAL_CSET_PATH}}
{% endif %}

[[install_website_skeleton]]
# Copies the static files that make up the web interface.
[[[environment]]]
CLEAN_WEB_DIR = {{CLEAN_WEB_DIR}}

[[fetch_fcst]]
# Fetch data from disk or a file based archival system.
inherit = FETCH_DATA
[[[environment]]]
ROSE_APP_OPT_CONF_KEYS = {{FETCH_FCST_OPT_CONF}}
CSET_INPUT_FILE_PATH = {{CSET_INPUT_FILE_PATH}}
{% if CSET_INCREMENTAL_DATA_FETCH %}
CSET_FILE_NAME_METADATA_PATTERN = {{CSET_FILE_NAME_METADATA_PATTERN}}
CSET_CYCLE_PERIOD = {{CSET_CYCLE_PERIOD}}
CSET_TIMES_PER_FILE = {{CSET_TIMES_PER_FILE}}
CSET_FILE_TIME_OFFSET = {{CSET_FILE_TIME_OFFSET}}
{% endif %}

[[housekeeping_raw]]
# Housekeep unprocessed data files.
script = rose task-run -v --app-key=housekeeping
[[[environment]]]
HOUSEKEEPING_MODE = {{[HOUSEKEEPING_MODE, 1]|min}}

[[housekeeping_full]]
# Housekeep processed intermediate files too.
script = rose task-run -v --app-key=housekeeping
[[[environment]]]
HOUSEKEEPING_MODE = {{[HOUSEKEEPING_MODE, 2]|min}}

[[finish_website]]
# Updates the workflow info in the web interface.
platform = localhost

[[send_email]]
# Send email to notify that the workflow is complete.
platform = localhost
[[[environment]]]
WEB_ADDR = {{WEB_ADDR}}


# Include files bring their own graph and runtime sections.
{% for include_file in glob("includes/*.cylc") %}
{% include include_file %}
{% endfor %}

{# Site-specific details that add to (or override) the core suite definition #}
{% include 'site/' ~ SITE ~ '.cylc' %}
Loading
Loading