CLI Only select specific variables

docs/source/batch_processing.rst

@@ -598,6 +598,34 @@ unaffected:
 
 The variable name(s) must appear in the config file's ``variables`` list.
 
+**Run only a specific subset of variables**
+
+Use ``--variable`` to limit a run to a specific subset of variables from the
+config, ignoring all others. This is useful for targeted first-runs or
+debugging a single variable without affecting the rest of the batch:
+
+.. code-block:: bash
+
+   # Run only Amon.tas
+   moppy-cmorise batch_config.yml --variable Amon.tas
+
+   # Run only a handful of variables
+   moppy-cmorise batch_config.yml --variable Amon.tas Amon.pr Omon.tos
+
+Only the listed variables are added to the tracking database for this
+invocation; variables not listed are neither inserted nor touched. The
+variable name(s) must appear in the config file's ``variables`` list.
+
+.. note::
+
+   ``--variable`` can be combined with ``--rerun-variable`` to limit the run
+   to a subset *and* force-reset one or more of those variables that may have
+   already completed:
+
+   .. code-block:: bash
+
+      moppy-cmorise batch_config.yml --variable Amon.tas --rerun-variable Amon.tas
+
 **Force re-run everything**
 
 ``--force`` resets every variable in the config (including completed ones) to

src/access_moppy/batch_cmoriser.py

@@ -623,6 +623,7 @@ def create_monitor_script(
     config_path: str | Path,
     db_path: str | Path,
     script_dir: str | Path,
+    variable_filter: list[str] | None = None,
 ) -> Path:
     """Render the PBS script for the monitor job and write it to script_dir.
 
@@ -644,6 +645,7 @@ def create_monitor_script(
         db_path=str(db_path),
         script_dir=str(script_dir),
         monitor_walltime=monitor_walltime,
+        variable_filter=variable_filter,
     )
 
     monitor_path = Path(script_dir) / "moppy_monitor.sh"
@@ -668,6 +670,7 @@ def monitor_main() -> None:
     config_path = os.environ.get("MOPPY_CONFIG_PATH")
     db_path = os.environ.get("MOPPY_DB_PATH")
     script_dir_env = os.environ.get("MOPPY_SCRIPT_DIR")
+    variable_filter_env = os.environ.get("MOPPY_VARIABLE_FILTER")
 
     if not config_path or not db_path:
         print(
@@ -689,6 +692,10 @@ def monitor_main() -> None:
     )
     script_dir.mkdir(parents=True, exist_ok=True)
 
+    variable_filter: set[str] | None = (
+        set(variable_filter_env.split(",")) if variable_filter_env else None
+    )
+
     tracker = TaskTracker(db_path)
     # Use try/finally so the sqlite handle is released on any exit path,
     # including the SystemExit raised by shutdown_handler.
@@ -697,6 +704,8 @@ def monitor_main() -> None:
         # processes. monitor_loop iterates this map to poll qstat per sub-job.
         job_map: dict[str, str] = {}
         for variable in config["variables"]:
+            if variable_filter is not None and variable not in variable_filter:
+                continue
             if tracker.is_done(variable, experiment_id):
                 print(f"Skipped (already completed): {variable}")
                 continue
@@ -936,6 +945,7 @@ def format_help(self) -> str:
             "  moppy-cmorise batch_config.yml\n"
             "  moppy-cmorise batch_config.yml --rerun-variable Amon.tas Amon.pr\n"
             "  moppy-cmorise batch_config.yml --force\n"
+            "  moppy-cmorise batch_config.yml --variable Amon.tas Amon.pr\n"
         ),
     )
     parser.add_argument(
@@ -963,6 +973,18 @@ def format_help(self) -> str:
             "submitting. Re-runs the entire batch from scratch."
         ),
     )
+    parser.add_argument(
+        "--variable",
+        metavar="VARIABLE",
+        nargs="+",
+        dest="variables",
+        default=None,
+        help=(
+            "Only run the specified variable(s) from the config, ignoring all others. "
+            "Useful for targeted first-runs or re-runs of specific variables. "
+            "Example: --variable Amon.tas Amon.pr"
+        ),
+    )
     return parser
 
 
@@ -996,7 +1018,16 @@ def main() -> None:
     with config_path.open() as f:
         config_data = yaml.safe_load(f)
 
-    # Validate --rerun-variable names against the config before touching the DB.
+    # Validate --variable and --rerun-variable names against the config before touching the DB.
+    if args.variables:
+        unknown = [v for v in args.variables if v not in config_data["variables"]]
+        if unknown:
+            print(
+                f"Error: --variable specifies variable(s) not in the config: "
+                f"{', '.join(unknown)}"
+            )
+            sys.exit(1)
+
     if args.rerun_variables:
         unknown = [v for v in args.rerun_variables if v not in config_data["variables"]]
         if unknown:
@@ -1013,17 +1044,22 @@ def main() -> None:
 
     experiment_id = config_data["experiment_id"]
 
-    # Pre-populate all tasks, then apply any forced resets so the monitor
-    # picks up the right set of variables to submit.
+    # Determine the effective variable list for this run.
+    active_variables = (
+        args.variables if args.variables else list(config_data["variables"])
+    )
+
+    # Pre-populate tasks for active variables, then apply any forced resets so
+    # the monitor picks up the right set of variables to submit.
     with TaskTracker(db_path) as tracker:
-        for variable in config_data["variables"]:
+        for variable in active_variables:
             tracker.add_task(variable, experiment_id)
 
         if args.force:
-            for variable in config_data["variables"]:
+            for variable in active_variables:
                 tracker.reset_to_pending(variable, experiment_id)
             print(
-                f"--force: reset {len(config_data['variables'])} variable(s) "
+                f"--force: reset {len(active_variables)} variable(s) "
                 "to pending (including any previously completed)."
             )
         elif args.rerun_variables:
@@ -1034,9 +1070,13 @@ def main() -> None:
                 f"to pending: {', '.join(args.rerun_variables)}"
             )
 
-    print(
-        f"Database initialized with {len(config_data['variables'])} tasks at: {db_path}"
-    )
+    if args.variables:
+        print(
+            f"Database initialized with {len(active_variables)} variable(s) "
+            f"(filtered from {len(config_data['variables'])} in config) at: {db_path}"
+        )
+    else:
+        print(f"Database initialized with {len(active_variables)} tasks at: {db_path}")
 
     # Start Streamlit dashboard (optional - won't block if streamlit is not installed)
     try:
@@ -1057,7 +1097,11 @@ def main() -> None:
     # reconciling DB state for any sub-job that exits without writing its own
     # terminal status (e.g. OOM-killed by PBS).
     monitor_script = create_monitor_script(
-        config_data, config_path, db_path, script_dir
+        config_data,
+        config_path,
+        db_path,
+        script_dir,
+        variable_filter=args.variables,
     )
     print(f"Created monitor script: {monitor_script}")
 
@@ -1071,7 +1115,7 @@ def main() -> None:
     sidecar.write_text(f"{monitor_job_id}\n{time.strftime('%Y-%m-%dT%H:%M:%S')}\n")
 
     print(f"\nSubmitted monitor job {monitor_job_id}")
-    print(f"  Watches {len(config_data['variables'])} variables")
+    print(f"  Watches {len(active_variables)} variable(s)")
     print(
         f"  Sub-jobs are qsub'd from the monitor (see {script_dir}/moppy_monitor.out)"
     )

src/access_moppy/templates/cmor_monitor_script.j2

@@ -21,7 +21,8 @@
 export MOPPY_CONFIG_PATH="{{ config_path }}"
 export MOPPY_DB_PATH="{{ db_path }}"
 export MOPPY_SCRIPT_DIR="{{ script_dir }}"
-# Defence in depth: QC must never run inside this 1-cpu/4GB job. It loads
+{% if variable_filter %}export MOPPY_VARIABLE_FILTER="{{ variable_filter | join(',') }}"
+{% endif %}# Defence in depth: QC must never run inside this 1-cpu/4GB job. It loads
 # full output files and scales with batch size (see finalize_monitor).
 export MOPPY_SKIP_QC=1