Record history in experiment, consolidate on experiment list write

SConscript

@@ -215,6 +215,7 @@ if not env_etc.no_boost_python and hasattr(env_etc, "boost_adaptbx_include"):
             "src/dxtbx/model/boost_python/crystal.cc",
             "src/dxtbx/model/boost_python/parallax_correction.cc",
             "src/dxtbx/model/boost_python/pixel_to_millimeter.cc",
+            "src/dxtbx/model/boost_python/history.cc",
             "src/dxtbx/model/boost_python/experiment.cc",
             "src/dxtbx/model/boost_python/experiment_list.cc",
             "src/dxtbx/model/boost_python/model_ext.cc",

newsfragments/816.feature

@@ -0,0 +1 @@
+Serialization history information is now stored with each ``Experiment``, and added to each time an ``ExperimentList`` is saved to disk.

src/dxtbx/CMakeLists.txt

@@ -39,6 +39,7 @@ Python_add_library( dxtbx_model_ext
     model/boost_python/crystal.cc
     model/boost_python/parallax_correction.cc
     model/boost_python/pixel_to_millimeter.cc
+    model/boost_python/history.cc
     model/boost_python/experiment.cc
     model/boost_python/experiment_list.cc
     model/boost_python/model_ext.cc )

src/dxtbx/model/__init__.py

@@ -1,16 +1,20 @@
 from __future__ import annotations
 
 import copy
+import importlib.metadata
+import inspect
 import json
 import os
 import sys
 
+import dateutil.parser
 from ordered_set import OrderedSet
 
 import boost_adaptbx.boost.python
 import cctbx.crystal
 import cctbx.sgtbx
 import cctbx.uctbx
+import libtbx.load_env
 from scitbx import matrix
 from scitbx.array_family import flex
 
@@ -36,6 +40,7 @@
         ExperimentType,
         Goniometer,
         GoniometerBase,
+        History,
         KappaDirection,
         KappaGoniometer,
         KappaScanAxis,
@@ -73,6 +78,7 @@
         ExperimentType,
         Goniometer,
         GoniometerBase,
+        History,
         KappaDirection,
         KappaGoniometer,
         KappaScanAxis,
@@ -599,6 +605,10 @@ def imagesets(self):
         """Get a list of the unique imagesets."""
         return list(OrderedSet([e.imageset for e in self if e.imageset is not None]))
 
+    def histories(self) -> list[History]:
+        """Get a list of the unique history objects."""
+        return list(OrderedSet([e.history for e in self if e.history is not None]))
+
     def all_stills(self):
         """Check if all the experiments are stills"""
         return all(exp.get_type() == ExperimentType.STILL for exp in self)
@@ -629,6 +639,30 @@ def all_same_type(self):
                 return False
         return True
 
+    def consolidate_histories(self) -> History:
+        """
+        Consolidate a list of histories into a single history and set this in each
+        experiment.
+
+        At the moment, this just combines the lines from the histories and sorts
+        them by timestamp.
+
+        :return History: The consolidated history
+        """
+        histories = self.histories()
+        if len(histories) == 0:
+            lines = []
+        else:
+            lines = [l for h in histories for l in h.get_history()]
+            lines.sort(key=lambda x: dateutil.parser.isoparse(x.split("|")[0]))
+        history = History(lines)
+
+        # Set the consolidated history in each experiment
+        for experiment in self:
+            experiment.history = history
+
+        return history
+
     def to_dict(self):
         """Serialize the experiment list to dictionary."""
 
@@ -660,10 +694,14 @@ def abspath_or_none(filename):
             for name, models, _ in lookup_members
         }
 
+        # If multiple histories are present, consolidate them
+        history = self.consolidate_histories()
+
         # Create the output dictionary
         result = {
             "__id__": "ExperimentList",
             "experiment": [],
+            "history": history.get_history(),
         }
 
         # Add the experiments to the dictionary
@@ -746,8 +784,71 @@ def nullify_all_single_file_reader_format_instances(self):
             if experiment.imageset.reader().is_single_file_reader():
                 experiment.imageset.reader().nullify_format_instance()
 
-    def as_json(self, filename=None, compact=False, split=False):
+    def as_json(
+        self,
+        filename=None,
+        compact=False,
+        split=False,
+        history_as_integrated=False,
+        history_as_scaled=False,
+    ):
         """Dump experiment list as json"""
+
+        # Find the module that called this function for the history
+        stack = inspect.stack()
+        this_module = inspect.getmodule(stack[0].frame)
+        caller_module_name = "Unknown"
+        for f in stack[1:]:
+            module = inspect.getmodule(f.frame)
+            if module != this_module and module is not None:
+                caller_module_name = module.__name__
+                break
+
+        # If that module was called directly, look up via file path
+        if caller_module_name == "__main__":
+            caller_module_name = os.path.splitext(os.path.basename(module.__file__))[0]
+
+        # Look up the dispatcher name for the caller module
+        try:
+            lookup = {e.module: e.name for e in importlib.metadata.entry_points()}
+        except AttributeError:  # Python < 3.10
+            lookup = {
+                e.module: e.name
+                for e in importlib.metadata.entry_points()["console_scripts"]
+            }
+        dispatcher = lookup.get(caller_module_name, caller_module_name)
+
+        # If dispatcher lookup by entry_points did not work, try via libtbx
+        if dispatcher == caller_module_name:
+            dispatcher = libtbx.env.dispatcher_name
+
+        # Final fallback to the module name
+        if dispatcher in ["dials.python", "libtbx.python", "cctbx.python", None]:
+            dispatcher = caller_module_name
+
+        # Get software version
+        try:
+            version = "v" + importlib.metadata.version(dispatcher.split(".")[0])
+        except importlib.metadata.PackageNotFoundError:
+            version = "v?"
+
+        # Set the flags string for the history
+        flags = []
+        if history_as_integrated:
+            flags.append("integrated")
+        if history_as_scaled:
+            flags.append("scaled")
+        if flags:
+            flags = ",".join(flags)
+        else:
+            flags = ""
+
+        # Consolidate existing history objects
+        history = self.consolidate_histories()
+
+        # Append the new history line
+        history.append_history_item(dispatcher, version, flags)
+
         # Get the dictionary and get the JSON string
         dictionary = self.to_dict()
 

src/dxtbx/model/boost_python/experiment.cc

@@ -31,7 +31,8 @@ namespace dxtbx { namespace model { namespace boost_python {
                                        obj.get_profile(),
                                        obj.get_imageset(),
                                        obj.get_scaling_model(),
-                                       obj.get_identifier());
+                                       obj.get_identifier(),
+                                       obj.get_history());
     }
   };
 
@@ -89,15 +90,18 @@ namespace dxtbx { namespace model { namespace boost_python {
                 boost::python::object,
                 boost::python::object,
                 boost::python::object,
-                std::string>((arg("beam") = std::shared_ptr<BeamBase>(),
-                              arg("detector") = std::shared_ptr<Detector>(),
-                              arg("goniometer") = std::shared_ptr<Goniometer>(),
-                              arg("scan") = std::shared_ptr<Scan>(),
-                              arg("crystal") = std::shared_ptr<CrystalBase>(),
-                              arg("profile") = boost::python::object(),
-                              arg("imageset") = boost::python::object(),
-                              arg("scaling_model") = boost::python::object(),
-                              arg("identifier") = "")))
+                std::string,
+                std::shared_ptr<History> >(
+        (arg("beam") = std::shared_ptr<BeamBase>(),
+         arg("detector") = std::shared_ptr<Detector>(),
+         arg("goniometer") = std::shared_ptr<Goniometer>(),
+         arg("scan") = std::shared_ptr<Scan>(),
+         arg("crystal") = std::shared_ptr<CrystalBase>(),
+         arg("profile") = boost::python::object(),
+         arg("imageset") = boost::python::object(),
+         arg("scaling_model") = boost::python::object(),
+         arg("identifier") = "",
+         arg("history") = std::shared_ptr<History>())))
       .add_property("beam", &Experiment::get_beam, &Experiment::set_beam)
       .add_property("detector", &Experiment::get_detector, &Experiment::set_detector)
       .add_property(
@@ -110,6 +114,7 @@ namespace dxtbx { namespace model { namespace boost_python {
         "scaling_model", &Experiment::get_scaling_model, &Experiment::set_scaling_model)
       .add_property(
         "identifier", &Experiment::get_identifier, &Experiment::set_identifier)
+      .add_property("history", &Experiment::get_history, &Experiment::set_history)
       .def("__contains__", experiment_contains_pointers::beam())
       .def("__contains__", experiment_contains_pointers::detector())
       .def("__contains__", experiment_contains_pointers::goniometer())

src/dxtbx/model/boost_python/history.cc

@@ -0,0 +1,43 @@
+#include <boost/python.hpp>
+#include <boost/python/def.hpp>
+#include <boost/python/args.hpp>
+#include <dxtbx/model/history.h>
+#include <dxtbx/error.h>
+
+namespace dxtbx { namespace model { namespace boost_python {
+
+  struct HistoryPickleSuite : boost::python::pickle_suite {
+    static boost::python::tuple getstate(boost::python::object obj) {
+      const History &history = boost::python::extract<const History &>(obj)();
+      return boost::python::make_tuple(obj.attr("__dict__"),
+                                       history.get_history_as_list());
+    }
+
+    static void setstate(boost::python::object obj, boost::python::tuple state) {
+      History &history = boost::python::extract<History &>(obj)();
+      DXTBX_ASSERT(boost::python::len(state) == 2);
+
+      // restore the object's __dict__
+      boost::python::dict d =
+        boost::python::extract<boost::python::dict>(obj.attr("__dict__"))();
+      d.update(state[0]);
+
+      // restore the internal state of the C++ object
+      history.set_history_from_list(
+        boost::python::extract<boost::python::list>(state[1])());
+    }
+  };
+
+  void export_history() {
+    using boost::python::arg;
+
+    boost::python::class_<History>("History")
+      .def(boost::python::init<const boost::python::list &>())
+      .def("set_history", &History::set_history_from_list)
+      .def("get_history", &History::get_history_as_list)
+      .def("append_history_item",
+           &History::append_history_item,
+           (arg("dispatcher"), arg("version"), arg("flag")))
+      .def_pickle(HistoryPickleSuite());
+  }
+}}}  // namespace dxtbx::model::boost_python

src/dxtbx/model/boost_python/model_ext.cc

@@ -29,6 +29,7 @@ namespace dxtbx { namespace model { namespace boost_python {
   void export_experiment();
   void export_experiment_list();
   void export_spectrum();
+  void export_history();
 
   BOOST_PYTHON_MODULE(dxtbx_model_ext) {
     export_beam();
@@ -45,6 +46,7 @@ namespace dxtbx { namespace model { namespace boost_python {
     export_experiment();
     export_experiment_list();
     export_spectrum();
+    export_history();
   }
 
 }}}  // namespace dxtbx::model::boost_python

src/dxtbx/model/experiment.h

@@ -24,6 +24,7 @@
 #include <dxtbx/model/goniometer.h>
 #include <dxtbx/model/scan.h>
 #include <dxtbx/model/crystal.h>
+#include <dxtbx/model/history.h>
 #include <dxtbx/error.h>
 
 namespace dxtbx { namespace model {
@@ -42,6 +43,8 @@ namespace dxtbx { namespace model {
    *   - crystal The crystal model
    *   - profile The profile model
    *   - scaling_model The scaling model
+   *   - identifier The experiment identifier
+   *   - history The serialization history of the experiment
    *
    * Some of these may be set to "None"
    *
@@ -61,7 +64,8 @@ namespace dxtbx { namespace model {
                boost::python::object profile,
                boost::python::object imageset,
                boost::python::object scaling_model,
-               std::string identifier)
+               std::string identifier,
+               std::shared_ptr<History> history)
         : beam_(beam),
           detector_(detector),
           goniometer_(goniometer),
@@ -70,7 +74,8 @@ namespace dxtbx { namespace model {
           profile_(profile),
           imageset_(imageset),
           scaling_model_(scaling_model),
-          identifier_(identifier) {}
+          identifier_(identifier),
+          history_(history) {}
 
     /**
      * Check if the beam model is the same.
@@ -248,6 +253,14 @@ namespace dxtbx { namespace model {
       return identifier_;
     }
 
+    void set_history(std::shared_ptr<History> history) {
+      history_ = history;
+    }
+
+    std::shared_ptr<History> get_history() const {
+      return history_;
+    }
+
   protected:
     std::shared_ptr<BeamBase> beam_;
     std::shared_ptr<Detector> detector_;
@@ -258,6 +271,7 @@ namespace dxtbx { namespace model {
     boost::python::object imageset_;
     boost::python::object scaling_model_;
     std::string identifier_;
+    std::shared_ptr<History> history_;
   };
 
 }}  // namespace dxtbx::model

src/dxtbx/model/experiment_list.py

@@ -30,6 +30,7 @@
     Experiment,
     ExperimentList,
     GoniometerFactory,
+    History,
     ProfileModelFactory,
     ScanFactory,
 )
@@ -510,6 +511,12 @@ def decode(self):
                 )
             )
 
+        # Add the history to the experiments if it exists
+        if "history" in self._obj:
+            history = History(self._obj["history"])
+            for expt in el:
+                expt.history = history
+
         return el
 
     def _make_mem_imageset(self, imageset):