Merge branch 'main' into ticket195

Author: rerpha

doc/architectural_decisions/006-where-to-put-code.md

@@ -25,7 +25,7 @@ Examples of devices and where we would put them under this model:
 
 ### `ibex_bluesky_core` devices
 - `BlockRw` and `SimpleDae`: `ibex_bluesky_core` as it's completely general / useful to all beamlines
-- `ReflParameter`: in `ibex_bluesky_core.devices.reflectrometry` because it's useful across all reflectometers
+- `ReflParameter`: in `ibex_bluesky_core.devices.reflectometry` because it's useful across all reflectometers
 - `Danfysik`: Try to push down "special" logic (i.e. polarity switching) into the IOC so that a standard `block_rw` works. Otherwise put in `ibex_bluesky_core.devices`
 
 ### `inst` scripts area devices

doc/dev/manual_system_testing.md

@@ -32,6 +32,10 @@ set "EPICS_CA_AUTO_ADDR_LIST=NO"
 set IBEX_BLUESKY_CORE_LOGS=c:\instrument\var\bluesky\tmp
 set IBEX_BLUESKY_CORE_OUTPUT=c:\instrument\var\bluesky\tmp
 ```
+- Checking the prerequisites:
+
+Ensure that any prerequisites are met before running the selected test. These can be located within the .py file.
+
 - Run the test using:
 ```
 python c:\instrument\dev\ibex_bluesky_core\manual_system_tests\the_test.py

doc/dev/troubleshooting.md

@@ -240,6 +240,8 @@ This is because the **datatype of the underlying PV** does not match the **decla
 will not allow you to connect a `block_r(str, "some_block")` if `"some_block"` is a float-type PV. Every signal in
 `ophyd_async` is strongly typed.
 
+If you're a developer, please ensure that you have followed the steps found on [`Manual System Testing`](/dev/manual_system_testing.md)
+
 ### Change how `set` on a device behaves
 
 For a {py:obj}`writable block<ibex_bluesky_core.devices.block.BlockRw>`, a `write_config` argument can be specified. 

src/ibex_bluesky_core/callbacks/__init__.py

@@ -21,7 +21,7 @@
     HumanReadableFileCallback,
 )
 from ibex_bluesky_core.callbacks._fitting import LiveFit, LiveFitLogger
-from ibex_bluesky_core.callbacks._plotting import LivePlot, show_plot
+from ibex_bluesky_core.callbacks._plotting import LivePlot, PlotPNGSaver, show_plot
 from ibex_bluesky_core.callbacks._utils import get_default_output_path
 from ibex_bluesky_core.fitting import FitMethod
 from ibex_bluesky_core.utils import is_matplotlib_backend_qt
@@ -38,6 +38,7 @@
     "LiveFit",
     "LiveFitLogger",
     "LivePlot",
+    "PlotPNGSaver",
     "get_default_output_path",
     "show_plot",
 ]
@@ -67,6 +68,9 @@ def __init__(  # noqa: PLR0912
         live_fit_logger_output_dir: str | PathLike[str] | None = None,
         live_fit_logger_postfix: str = "",
         human_readable_file_postfix: str = "",
+        save_plot_to_png: bool = True,
+        plot_png_output_dir: str | PathLike[str] | None = None,
+        plot_png_postfix: str = "",
         live_fit_update_every: int | None = 1,
         live_plot_update_on_every_event: bool = True,
     ) -> None:
@@ -127,9 +131,13 @@ def _inner():
             live_fit_logger_output_dir: the output directory for live fit logger.
             live_fit_logger_postfix: the postfix to add to live fit logger.
             human_readable_file_postfix: optional postfix to add to human-readable file logger.
+            save_plot_to_png: whether to save the plot to a PNG file.
+            plot_png_output_dir: the output directory for plotting PNG files.
+            plot_png_postfix: the postfix to add to PNG plot files.
             live_fit_update_every: How often, in points, to recompute the fit. If None, do not compute until the end.
             live_plot_update_on_every_event: whether to show the live plot on every event, or just at the end.
         """  # noqa
+        fig = None
         self._subs = []
         self._peak_stats = None
         self._live_fit = None
@@ -230,6 +238,16 @@ def start(self, doc: RunStart) -> None:
                     update_on_every_event=live_plot_update_on_every_event,
                 )
             )
+            if save_plot_to_png and ax is not None:
+                self._subs.append(
+                    PlotPNGSaver(
+                        x=x,
+                        y=y,
+                        ax=ax,
+                        output_dir=plot_png_output_dir,
+                        postfix=plot_png_postfix,
+                    )
+                )
 
     @property
     def live_fit(self) -> LiveFit:

src/ibex_bluesky_core/callbacks/_file_logger.py

@@ -3,9 +3,7 @@
 import csv
 import logging
 import os
-from datetime import datetime
 from pathlib import Path
-from zoneinfo import ZoneInfo
 
 from bluesky.callbacks import CallbackBase
 from event_model.documents.event import Event
@@ -20,13 +18,13 @@
     MOTORS,
     NAME,
     PRECISION,
-    RB,
     SEQ_NUM,
     START_TIME,
     TIME,
     UID,
     UNITS,
-    UNKNOWN_RB,
+    _get_rb_num,
+    format_time,
     get_default_output_path,
     get_instrument,
 )
@@ -68,23 +66,19 @@ def start(self, doc: RunStart) -> None:
         self.output_dir.mkdir(parents=True, exist_ok=True)
         self.current_start_document = doc[UID]
 
-        datetime_obj = datetime.fromtimestamp(doc[TIME])
-        title_format_datetime = datetime_obj.astimezone(ZoneInfo("UTC")).strftime(
-            "%Y-%m-%d_%H-%M-%S"
-        )
-        rb_num = doc.get(RB, UNKNOWN_RB)
+        rb_num = _get_rb_num(doc)
 
         # motors is a tuple, we need to convert to a list to join the two below
         motors = list(doc.get(MOTORS, []))
 
+        formatted_time = format_time(doc)
+
         self.filename = (
             self.output_dir
             / f"{rb_num}"
             / f"{get_instrument()}{'_' + '_'.join(motors) if motors else ''}_"
-            f"{title_format_datetime}Z{self.postfix}.txt"
+            f"{formatted_time}Z{self.postfix}.txt"
         )
-        if rb_num == UNKNOWN_RB:
-            logger.warning('No RB number found, saving to "%s"', UNKNOWN_RB)
         assert self.filename is not None
         logger.info("starting new file %s", self.filename)
 
@@ -93,7 +87,6 @@ def start(self, doc: RunStart) -> None:
         ]
         header_data = {k: v for k, v in doc.items() if k not in exclude_list}
 
-        formatted_time = datetime_obj.astimezone(ZoneInfo("UTC")).strftime("%Y-%m-%d %H:%M:%S")
         header_data[START_TIME] = formatted_time
 
         # make sure the parent directory exists, create it if not

src/ibex_bluesky_core/callbacks/_fitting.py

@@ -4,9 +4,7 @@
 import logging
 import os
 import warnings
-from datetime import datetime
 from pathlib import Path
-from zoneinfo import ZoneInfo
 
 import numpy as np
 from bluesky.callbacks import CallbackBase
@@ -16,10 +14,9 @@
 
 from ibex_bluesky_core.callbacks._utils import (
     DATA,
-    RB,
-    TIME,
     UID,
-    UNKNOWN_RB,
+    _get_rb_num,
+    format_time,
     get_default_output_path,
     get_instrument,
 )
@@ -167,16 +164,13 @@ def start(self, doc: RunStart) -> None:
             doc (RunStart): The start bluesky document.
 
         """
-        datetime_obj = datetime.fromtimestamp(doc[TIME])
-        title_format_datetime = datetime_obj.astimezone(ZoneInfo("UTC")).strftime(
-            "%Y-%m-%d_%H-%M-%S"
-        )
+        title_format_datetime = format_time(doc)
         self.output_dir.mkdir(parents=True, exist_ok=True)
         self.current_start_document = doc[UID]
         file = f"{get_instrument()}_{self.x}_{self.y}_{title_format_datetime}Z{self.postfix}.txt"
-        rb_num = doc.get(RB, UNKNOWN_RB)
-        if rb_num == UNKNOWN_RB:
-            logger.warning('No RB number found, will save to "%s"', UNKNOWN_RB)
+
+        rb_num = _get_rb_num(doc)
+
         self.filename = self.output_dir / f"{rb_num}" / file
 
     def event(self, doc: Event) -> Event:

src/ibex_bluesky_core/callbacks/_plotting.py

@@ -1,18 +1,29 @@
 """IBEX plotting callbacks."""
 
 import logging
+import os
+from pathlib import Path
 from typing import Any
 
 import matplotlib
 import matplotlib.pyplot as plt
 from bluesky.callbacks import LivePlot as _DefaultLivePlot
 from bluesky.callbacks.core import get_obj_fields, make_class_safe
+from bluesky.callbacks.mpl_plotting import QtAwareCallback
 from event_model import RunStop
 from event_model.documents import Event, RunStart
+from matplotlib.axes import Axes
+
+from ibex_bluesky_core.callbacks._utils import (
+    _get_rb_num,
+    format_time,
+    get_default_output_path,
+    get_instrument,
+)
 
 logger = logging.getLogger(__name__)
 
-__all__ = ["LivePlot", "show_plot"]
+__all__ = ["LivePlot", "PlotPNGSaver", "show_plot"]
 
 
 def show_plot() -> None:
@@ -98,3 +109,53 @@ def stop(self, doc: RunStop) -> None:
         if not self.update_on_every_event:
             self.update_plot(force=True)
             show_plot()
+
+
+class PlotPNGSaver(QtAwareCallback):
+    """Save plots to PNG files on a run end."""
+
+    def __init__(
+        self,
+        x: str,
+        y: str,
+        ax: Axes,
+        postfix: str,
+        output_dir: str | os.PathLike[str] | None,
+    ) -> None:
+        """Initialise the PlotPNGSaver callback.
+
+        Args:
+            x: The name of the signal for x.
+            y: The name of the signal for y.
+            ax: The subplot to save to a file.
+            postfix: The file postfix.
+            output_dir: The output directory for PNGs.
+
+        """
+        super().__init__()
+        self.x = x
+        self.y = y
+        self.ax = ax
+        self.postfix = postfix
+        self.output_dir = Path(output_dir or get_default_output_path())
+        self.filename = None
+
+    def start(self, doc: RunStart) -> None:
+        self.filename = (
+            self.output_dir
+            / f"{_get_rb_num(doc)}"
+            / f"{get_instrument()}_{self.x}_{self.y}_{format_time(doc)}Z{self.postfix}.png"
+        )
+
+    def stop(self, doc: RunStop) -> None:
+        """Write the current plot to a PNG file.
+
+        Args:
+            doc: The stop document.
+
+        """
+        if self.filename is None:
+            raise ValueError("No filename specified for plot PNG")
+
+        self.filename.parent.mkdir(parents=True, exist_ok=True)
+        self.ax.figure.savefig(self.filename, format="png")  # pyright: ignore [reportAttributeAccessIssue]

src/ibex_bluesky_core/callbacks/_utils.py

@@ -1,6 +1,13 @@
+import logging
 import os
+from datetime import datetime
 from pathlib import Path
 from platform import node
+from zoneinfo import ZoneInfo
+
+from event_model import Event, RunStart, RunStop
+
+logger = logging.getLogger(__name__)
 
 OUTPUT_DIR_ENV_VAR = "IBEX_BLUESKY_CORE_OUTPUT"
 
@@ -31,3 +38,16 @@ def get_default_output_path() -> Path:
         if output_dir_env is None
         else Path(output_dir_env)
     )
+
+
+def format_time(doc: Event | RunStart | RunStop) -> str:
+    datetime_obj = datetime.fromtimestamp(doc[TIME])
+    title_format_datetime = datetime_obj.astimezone(ZoneInfo("UTC")).strftime("%Y-%m-%d_%H-%M-%S")
+    return title_format_datetime
+
+
+def _get_rb_num(doc: Event | RunStart | RunStop) -> str:
+    rb_num = doc.get(RB, UNKNOWN_RB)
+    if rb_num == UNKNOWN_RB:
+        logger.warning('No RB number found, saving to "%s"', UNKNOWN_RB)
+    return rb_num

src/ibex_bluesky_core/devices/reflectometry.py

@@ -101,17 +101,22 @@ async def set(self, value: float) -> None:
         await asyncio.sleep(1.0)
 
 
-def refl_parameter(name: str, changing_timeout_s: float = 60.0) -> ReflParameter:
+def refl_parameter(
+    name: str, changing_timeout_s: float = 60.0, has_redefine: bool = True
+) -> ReflParameter:
     """Small wrapper around a reflectometry parameter device.
 
     This automatically applies the current instrument's PV prefix.
 
     Args:
         name: the reflectometry parameter name.
         changing_timeout_s: time to wait (seconds) for the CHANGING signal to go False after a set.
+        has_redefine: whether this parameter can be redefined.
 
     Returns a device pointing to a reflectometry parameter.
 
     """
     prefix = get_pv_prefix()
-    return ReflParameter(prefix=prefix, name=name, changing_timeout_s=changing_timeout_s)
+    return ReflParameter(
+        prefix=prefix, name=name, changing_timeout_s=changing_timeout_s, has_redefine=has_redefine
+    )

src/ibex_bluesky_core/devices/simpledae/_waiters.py

@@ -8,6 +8,7 @@
 from ophyd_async.core import (
     Device,
     SignalR,
+    soft_signal_rw,
     wait_for_value,
 )
 
@@ -32,13 +33,14 @@ def __init__(self, value: T) -> None:
             value: the value to wait for
 
         """
-        self._value: T = value
+        self.finish_wait_at = soft_signal_rw(float, value)
 
     async def wait(self, dae: "SimpleDae") -> None:
         """Wait for signal to reach the user-specified value."""
         signal = self.get_signal(dae)
         logger.info("starting wait for signal %s", signal.source)
-        await wait_for_value(signal, lambda v: v >= self._value, timeout=None)
+        value = await self.finish_wait_at.get_value()
+        await wait_for_value(signal, lambda v: v >= value, timeout=None)
         logger.info("completed wait for signal %s", signal.source)
 
     def additional_readable_signals(self, dae: "SimpleDae") -> list[Device]: