[MAINT, MRG] Enable pooch to perform fetching of datasets (mne-tools#9742)

* Adding updated checksum

* Initial try

* Fix import

* Fix check

* Fix GH Actions

* Try again to fix CI

* make brainstorm work

* Address comments initially

* Removed _fetch_file dependency

* Remove fetch file entirely

* Lets go again with updated deletion

* Try again with testing reqs

* Try again

* Fix test constants

* Fix docs

* fix unclosed resource warning (checksum registry)

* make compatible with py3.7

* fix CIs?

* Fixed all unit tests

* Fix spell check

* Fix sys

* Adding updated checksum

* Initial try

* Fix import

* Fix check

* Fix GH Actions

* Try again to fix CI

* make brainstorm work

* Address comments initially

* Removed _fetch_file dependency

* Remove fetch file entirely

* Lets go again with updated deletion

* Try again with testing reqs

* Try again

* Fix test constants

* Fix docs

* fix unclosed resource warning (checksum registry)

* make compatible with py3.7

* fix CIs?

* Fixed all unit tests

* Fix spell check

* Fix sys

* purge importlib_resources and add _resource_path util function

* Apply suggestions from code review

* Fix flake

* Update requirements_testing.txt

* restore pooch to testing reqs

* Nested import for pooch

* Add back in

* Fix resoure warning

Co-authored-by: Daniel McCloy <dan@mccloy.info>

Author: adam2392

MANIFEST.in

@@ -21,6 +21,8 @@ recursive-include mne/data/fsaverage *
 include mne/datasets/_fsaverage/root.txt
 include mne/datasets/_fsaverage/bem.txt
 include mne/datasets/_infant/*.txt
+include mne/data/dataset_checksums.txt
+include mne/data/eegbci_checksums.txt
 
 recursive-include mne/channels/data/layouts *
 recursive-include mne/channels/data/montages *

doc/changes/latest.inc

@@ -165,6 +165,8 @@ Enhancements
 
 - Add :func:`mne.gui.locate_ieeg` to locate intracranial electrode contacts from a CT, an MRI (with Freesurfer ``recon-all``) and the channel names from an :class:`mne.Info` object (:gh:`9586` by `Alex Rockhill`_)
 
+- All data fetching code now relies on ``pooch``, which is an added optional requirement for dataset fetchers (:gh:`9742` by `Adam Li`_ and `Daniel McCloy`_)
+
 Bugs
 ~~~~
 - Fix bug in :meth:`mne.io.Raw.pick` and related functions when parameter list contains channels which are not in info instance (:gh:`9708` **by new contributor** |Evgeny Goldstein|_)

environment.yml

@@ -37,5 +37,6 @@ dependencies:
 - mne
 - mffpy>=0.5.7
 - ipywidgets
+- pooch
 - pip:
   - ipyvtklink

mne/conftest.py

@@ -613,7 +613,8 @@ def _fail(*args, **kwargs):
 @pytest.fixture(scope='function')
 def download_is_error(monkeypatch):
     """Prevent downloading by raising an error when it's attempted."""
-    monkeypatch.setattr(mne.utils.fetching, '_get_http', _fail)
+    import pooch
+    monkeypatch.setattr(pooch, 'retrieve', _fail)
 
 
 @pytest.fixture()

mne/data/dataset_checksums.txt

@@ -0,0 +1,46 @@
+# include here the name of the zipped file and the md5 hash
+# Note: other hashing algorithms can be enforced
+MNE-fNIRS-motor-data.tgz                   md5:c4935d19ddab35422a69f3326a01fef8
+MNE-kiloword-data.tar.gz                   md5:3a124170795abbd2e48aae8727e719a8
+MNE-multimodal-data.tar.gz                 md5:26ec847ae9ab80f58f204d09e2c08367
+MNE-OPM-data.tar.gz                        md5:370ad1dcfd5c47e029e692c85358a374
+MNE-phantom-4DBTi.zip                      md5:938a601440f3ffa780d20a17bae039ff
+MNE-sample-data-processed.tar.gz           md5:12b75d1cb7df9dfb4ad73ed82f61094f
+MNE-somato-data.tar.gz                     md5:32fd2f6c8c7eb0784a1de6435273c48b
+MNE-spm-face.tar.gz                        md5:9f43f67150e3b694b523a21eb929ea75
+MNE-visual_92_categories-data-part1.tar.gz md5:74f50bbeb65740903eadc229c9fa759f
+MNE-visual_92_categories-data-part2.tar.gz md5:203410a98afc9df9ae8ba9f933370e20
+mTRF_1.5.zip                               md5:273a390ebbc48da2c3184b01a82e4636
+sample_reference_MEG_noise-raw.zip         md5:779fecd890d98b73a4832e717d7c7c45
+ssvep_example_data.zip                     md5:af866bbc0f921114ac9d683494fe87d6
+MNE-ERP-CORE-data.tar.gz                   md5:5866c0d6213bd7ac97f254c776f6c4b1
+MNE-epilepsy-ecog-data.tar.gz              md5:ffb139174afa0f71ec98adbbb1729dea
+
+# Fieldtrip CMC dataset
+SubjectCMC.zip                             md5:6f9fd6520f9a66e20994423808d2528c
+
+# brainstorm datasets:
+bst_auditory.tar.gz                        md5:fa371a889a5688258896bfa29dd1700b
+bst_phantom_ctf.tar.gz                     md5:80819cb7f5b92d1a5289db3fb6acb33c
+bst_phantom_elekta.tar.gz                  md5:1badccbe17998d18cc373526e86a7aaf
+bst_raw.tar.gz                             md5:fa2efaaec3f3d462b319bc24898f440c
+bst_resting.tar.gz                         md5:70fc7bf9c3b97c4f2eab6260ee4a0430
+
+# HF-SEF
+hf_sef_raw.tar.gz                          md5:33934351e558542bafa9b262ac071168
+hf_sef_evoked.tar.gz                       md5:13d34cb5db584e00868677d8fb0aab2b
+
+# "fake" dataset (for testing)
+foo.tgz                                    md5:3194e9f7b46039bb050a74f3e1ae9908
+
+# FreeSurfer files
+lh.aparc_sub.annot                         md5:9e4d8d6b90242b7e4b0145353436ef77
+rh.aparc_sub.annot                         md5:dd6464db8e7762d969fc1d8087cd211b
+
+# below, do **not** include the release
+# number in the names of the testing or
+# misc datasets (it gets done automatically
+# in mne/datasets/utils.py). Only update
+# the md5 hashes as needed.
+mne-testing-data                           md5:db07710c0b94476f954f60926685b5b7
+mne-misc-data                              md5:0aa25a9bb4f204b3d4769f0b84e9b526

mne/data/eegbci_checksums.txt

@@ -8,7 +8,7 @@
 from ..utils import (has_dataset, _data_path, _get_version, _version_doc,
                      _data_path_doc_accept)
 
-has_brainstorm_data = partial(has_dataset, name='brainstorm.bst_auditory')
+has_brainstorm_data = partial(has_dataset, name='bst_auditory')
 
 
 _description = u"""
@@ -31,9 +31,8 @@
 def data_path(path=None, force_update=False, update_path=True, download=True,
               *, accept=False, verbose=None):  # noqa: D103
     return _data_path(path=path, force_update=force_update,
-                      update_path=update_path, name='brainstorm',
-                      download=download, archive_name='bst_auditory.tar.gz',
-                      accept=accept)
+                      update_path=update_path, name='bst_auditory',
+                      download=download, accept=accept)
 
 
 _data_path_doc = _data_path_doc_accept.format(
@@ -44,7 +43,7 @@ def data_path(path=None, force_update=False, update_path=True, download=True,
 
 
 def get_version():  # noqa: D103
-    return _get_version('brainstorm.bst_auditory')
+    return _get_version('bst_auditory')
 
 
 get_version.__doc__ = _version_doc.format(name='brainstorm')

mne/datasets/brainstorm/bst_auditory.py

@@ -8,7 +8,7 @@
 from ..utils import (has_dataset, _data_path, _get_version, _version_doc,
                      _data_path_doc_accept)
 
-has_brainstorm_data = partial(has_dataset, name='brainstorm.bst_phantom_ctf')
+has_brainstorm_data = partial(has_dataset, name='bst_phantom_ctf')
 
 
 _description = u"""
@@ -20,9 +20,8 @@
 def data_path(path=None, force_update=False, update_path=True, download=True,
               *, accept=False, verbose=None):  # noqa: D103
     return _data_path(path=path, force_update=force_update,
-                      update_path=update_path, name='brainstorm',
-                      download=download, archive_name='bst_phantom_ctf.tar.gz',
-                      accept=accept)
+                      update_path=update_path, name='bst_phantom_ctf',
+                      download=download, accept=accept)
 
 
 _data_path_doc = _data_path_doc_accept.format(
@@ -33,7 +32,7 @@ def data_path(path=None, force_update=False, update_path=True, download=True,
 
 
 def get_version():  # noqa: D103
-    return _get_version('brainstorm.bst_phantom_ctf')
+    return _get_version('bst_phantom_ctf')
 
 
 get_version.__doc__ = _version_doc.format(name='brainstorm')

mne/datasets/brainstorm/bst_phantom_ctf.py

@@ -9,7 +9,7 @@
                      _data_path_doc_accept)
 
 has_brainstorm_data = partial(has_dataset,
-                              name='brainstorm.bst_phantom_elekta')
+                              name='bst_phantom_elekta')
 
 
 _description = u"""
@@ -21,10 +21,8 @@
 def data_path(path=None, force_update=False, update_path=True, download=True,
               *, accept=False, verbose=None):  # noqa: D103
     return _data_path(path=path, force_update=force_update,
-                      update_path=update_path, name='brainstorm',
-                      download=download,
-                      archive_name='bst_phantom_elekta.tar.gz',
-                      accept=accept)
+                      update_path=update_path, name='bst_phantom_elekta',
+                      download=download, accept=accept)
 
 
 _data_path_doc = _data_path_doc_accept.format(
@@ -36,7 +34,7 @@ def data_path(path=None, force_update=False, update_path=True, download=True,
 
 
 def get_version():  # noqa: D103
-    return _get_version('brainstorm.bst_phantom_elekta')
+    return _get_version('bst_phantom_elekta')
 
 
 get_version.__doc__ = _version_doc.format(name='brainstorm')

mne/datasets/brainstorm/bst_phantom_elekta.py

@@ -8,7 +8,7 @@
 from ..utils import (has_dataset, _data_path, _get_version, _version_doc,
                      _data_path_doc_accept)
 
-has_brainstorm_data = partial(has_dataset, name='brainstorm.bst_raw')
+has_brainstorm_data = partial(has_dataset, name='bst_raw')
 
 _description = u"""
 URL: http://neuroimage.usc.edu/brainstorm/DatasetMedianNerveCtf
@@ -29,9 +29,8 @@
 def data_path(path=None, force_update=False, update_path=True, download=True,
               *, accept=False, verbose=None):    # noqa: D103
     return _data_path(path=path, force_update=force_update,
-                      update_path=update_path, name='brainstorm',
-                      download=download, archive_name='bst_raw.tar.gz',
-                      accept=accept)
+                      update_path=update_path, name='bst_raw',
+                      download=download, accept=accept)
 
 
 _data_path_doc = _data_path_doc_accept.format(
@@ -42,7 +41,7 @@ def data_path(path=None, force_update=False, update_path=True, download=True,
 
 
 def get_version():  # noqa: D103
-    return _get_version('brainstorm.bst_raw')
+    return _get_version('bst_raw')
 
 
 get_version.__doc__ = _version_doc.format(name='brainstorm')