Merge pull request #457 from ImperialCollegeLondon/354-xarrayopen_mfd…

…ataset-can-cause-problems-in-vr_run

Turning on single-threaded handling in dask

virtual_ecosystem/core/data.py

@@ -123,6 +123,7 @@
 from pathlib import Path
 from typing import Any
 
+import dask
 import numpy as np
 from xarray import DataArray, Dataset, open_mfdataset
 
@@ -133,6 +134,16 @@
 from virtual_ecosystem.core.readers import load_to_dataarray
 from virtual_ecosystem.core.utils import check_outfile
 
+# There are ongoing xarray issues with NetCDF not being thread safe and this causes
+# segfaults on different architectures in testing using `xarray.open_mfdataset`
+# See:
+# - https://github.com/pydata/xarray/issues/7079
+# - https://github.com/pydata/xarray/issues/3961
+#
+# Following advice on both those issues, we currently explicitly stop dask from trying
+# to use parallel file processing and use open_mfdataset(..., lock=False)
+dask.config.set(scheduler="single-threaded")
+
 
 class Data:
     """The Virtual Ecosystem data object.