Merge pull request #200 from python-adaptive/safe_saving

ensure atomic writes when saving a file

Author: jbweston

adaptive/utils.py

@@ -7,6 +7,8 @@
 from contextlib import contextmanager
 from itertools import product
 
+from atomicwrites import AtomicWriter
+
 
 def named_product(**items):
     names = items.keys()
@@ -44,9 +46,13 @@ def save(fname, data, compress=True):
     dirname = os.path.dirname(fname)
     if dirname:
         os.makedirs(dirname, exist_ok=True)
-    _open = gzip.open if compress else open
-    with _open(fname, "wb") as f:
-        pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+    blob = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)
+    if compress:
+        blob = gzip.compress(blob)
+
+    with AtomicWriter(fname, "wb", overwrite=True).open() as f:
+        f.write(blob)
 
 
 def load(fname, compress=True):

docs/environment.yml

@@ -12,6 +12,7 @@ dependencies:
   - bokeh=1.0.4
   - plotly=3.9.0
   - ipywidgets=7.4.2
+  - atomicwrites=1.3.0
   - pip:
     - git+https://github.com/basnijholt/jupyter-sphinx.git@widgets_execute
     - sphinx_fontawesome==0.0.6

environment.yml

@@ -16,3 +16,4 @@ dependencies:
   - ipywidgets
   - scikit-optimize
   - plotly
+  - atomicwrites

setup.py

@@ -24,7 +24,12 @@ def get_version_and_cmdclass(package_name):
 version, cmdclass = get_version_and_cmdclass("adaptive")
 
 
-install_requires = ["scipy", "sortedcollections >= 1.1", "sortedcontainers >= 2.0"]
+install_requires = [
+    "scipy",
+    "sortedcollections >= 1.1",
+    "sortedcontainers >= 2.0",
+    "atomicwrites",
+]
 
 extras_require = {
     "notebook": [