Skip to content

Commit 3e36b0f

Browse files
larsonerlarsoner
committed
Merge remote-tracking branch 'upstream/master' into tf
* upstream/master: MAINT: Actually use all caches (mne-tools#8702) MRG: Prototype of notebook viz (ipyvtk) (mne-tools#8503) MRG: Use caching in Github Actions and Azure Pipelines (mne-tools#8695)
2 parents df6f82a + 68e02f5 commit 3e36b0f

19 files changed

+406
-389
lines changed

‎.circleci/config.yml‎

Lines changed: 137 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,21 @@ jobs:
66
docker:
77
- image: circleci/python:3.8.5-buster
88
steps:
9+
- restore_cache:
10+
keys:
11+
- source-cache
912
- checkout
13+
- run:
14+
name: Complete checkout
15+
command: |
16+
if ! git remote -v | grep upstream; then
17+
git remote add upstream git://github.com/mne-tools/mne-python.git
18+
fi
19+
git fetch upstream
20+
- save_cache:
21+
key: source-cache
22+
paths:
23+
- ".git"
1024
- run:
1125
name: Set BASH_ENV
1226
command: |
@@ -21,6 +35,8 @@ jobs:
2135
echo "export PATH=~/.local/bin/:${MNE_ROOT}/bin:$PATH" >> $BASH_ENV
2236
echo "BASH_ENV:"
2337
cat $BASH_ENV
38+
mkdir -p ~/mne_data
39+
touch pattern.txt;
2440
- run:
2541
name: check neuromag2ft
2642
command: |
@@ -33,25 +49,9 @@ jobs:
3349
echo ${CI_PULL_REQUEST//*pull\//} | tee merge.txt
3450
if [[ $(cat merge.txt) != "" ]]; then
3551
echo "Merging $(cat merge.txt)";
36-
git remote add upstream git://github.com/mne-tools/mne-python.git;
3752
git pull --ff-only upstream "refs/pull/$(cat merge.txt)/merge";
38-
git fetch upstream master;
3953
fi
4054
41-
# Load our data
42-
- restore_cache:
43-
keys:
44-
- data-cache-0
45-
- data-cache-1
46-
- data-cache-2
47-
- data-cache-3
48-
- data-cache-4
49-
- data-cache-5
50-
- data-cache-6
51-
- data-cache-7
52-
- data-cache-8
53-
- pip-cache
54-
5555
- run:
5656
name: Install 3D rendering libraries \ PyQt5 dependencies \ graphviz \ optipng (for optimized images)
5757
command: |
@@ -76,26 +76,44 @@ jobs:
7676
curl https://codeload.github.com/adobe-fonts/source-sans-pro/tar.gz/3.006R | tar xz -C $HOME/.fonts
7777
fc-cache -f
7878
79+
# Load pip cache
80+
- restore_cache:
81+
keys:
82+
- pip-cache
83+
- restore_cache:
84+
keys:
85+
- user-install-bin-cache
86+
7987
- run:
8088
name: Get Python running
8189
command: |
8290
python -m pip install --user --upgrade --progress-bar off pip setuptools
8391
python -m pip install --user --upgrade --progress-bar off --pre sphinx
92+
python -m pip install --user --upgrade --progress-bar off -r requirements.txt -r requirements_testing.txt -r requirements_doc.txt
8493
python -m pip install --user --progress-bar off https://github.com/pyvista/pyvista/zipball/master
8594
python -m pip install --user --progress-bar off https://github.com/pyvista/pyvistaqt/zipball/master
86-
python -m pip install --user --upgrade --progress-bar off -r requirements.txt -r requirements_testing.txt -r requirements_doc.txt
8795
python -m pip uninstall -yq pysurfer mayavi
8896
python -m pip install --user -e .
8997
9098
- save_cache:
9199
key: pip-cache
92100
paths:
93101
- ~/.cache/pip
102+
- save_cache:
103+
key: user-install-bin-cache
104+
paths:
105+
- ~/.local/lib/python3.8/site-packages
106+
- ~/.local/bin
94107

95108
- run:
96109
name: Check PyQt5
97110
command: LD_DEBUG=libs python -c "from PyQt5.QtWidgets import QApplication, QWidget; app = QApplication([])"
98111

112+
# Load tiny cache so that ~/.mne does not need to be created below
113+
- restore_cache:
114+
keys:
115+
- data-cache-tiny-0
116+
99117
# Look at what we have and fail early if there is some library conflict
100118
- run:
101119
name: Check installation
@@ -109,14 +127,55 @@ jobs:
109127
python -c "import mne; level = mne.get_config('MNE_LOGGING_LEVEL'); assert level.lower() == 'info', repr(level)"
110128
111129
# Figure out if we should run a full, pattern, or noplot version
130+
- restore_cache:
131+
keys:
132+
- data-cache-multimodal
133+
- restore_cache:
134+
keys:
135+
- data-cache-limo
136+
- restore_cache:
137+
keys:
138+
- data-cache-bst-phantom-ctf
139+
- restore_cache:
140+
keys:
141+
- data-cache-bst-raw
142+
- restore_cache:
143+
keys:
144+
- data-cache-bst-phantom-elekta
145+
- restore_cache:
146+
keys:
147+
- data-cache-bst-auditory
148+
- restore_cache:
149+
keys:
150+
- data-cache-bst-resting
151+
- restore_cache:
152+
keys:
153+
- data-cache-fieldtrip
154+
- restore_cache:
155+
keys:
156+
- data-cache-somato
157+
- restore_cache:
158+
keys:
159+
- data-cache-hf-sef
160+
- restore_cache:
161+
keys:
162+
- data-cache-opm
163+
- restore_cache:
164+
keys:
165+
- data-cache-sample
166+
- restore_cache:
167+
keys:
168+
- data-cache-spm-face
169+
- restore_cache:
170+
keys:
171+
- data-cache-testing
172+
- restore_cache:
173+
keys:
174+
- data-cache-visual
112175
- run:
113176
name: Get data
114177
command: |
115-
python setup.py develop --user
116-
mkdir -p ~/mne_data
117-
touch pattern.txt;
118178
./tools/circleci_download.sh
119-
120179
- run:
121180
name: Get data (again)
122181
when: on_fail
@@ -165,15 +224,11 @@ jobs:
165224
name: Reduce artifact upload time
166225
command: |
167226
if grep -q html_dev-pattern-memory build.txt || grep -q html_dev-noplot build.txt; then
168-
tar czf doc/_build/html/_downloads.tgz doc/_build/html/_downloads
169-
rm -Rf doc/_build/html/_downloads
170-
rm -f doc/auto_*/*/*.pickle
171-
rm -f doc/auto_*/*/*.codeobj
172-
rm -f doc/auto_*/*/*.md5
173-
rm -f doc/auto_*/*/*.py
174-
rm -f doc/auto_*/*/*.ipynb
175-
rm -f doc/generated/*.examples
227+
zip -rm doc/_build/html/_downloads.zip doc/_build/html/_downloads
176228
fi
229+
for NAME in generated auto_tutorials auto_examples; do
230+
zip -rm doc/${NAME}.zip doc/${NAME}
231+
done
177232
178233
# Save the JUnit file
179234
- store_test_results:
@@ -183,14 +238,11 @@ jobs:
183238
destination: test-results
184239
# Save the SG RST
185240
- store_artifacts:
186-
path: doc/auto_examples
187-
destination: auto_examples
241+
path: doc/auto_examples.zip
188242
- store_artifacts:
189-
path: doc/auto_tutorials
190-
destination: auto_tutorials
243+
path: doc/auto_tutorials.zip
191244
- store_artifacts:
192-
path: doc/generated
193-
destination: generated
245+
path: doc/generated.zip
194246
# Save the HTML
195247
- store_artifacts:
196248
path: doc/_build/html/
@@ -206,50 +258,74 @@ jobs:
206258

207259
# Keep these separate, maybe better in terms of size limitations (?)
208260
- save_cache:
209-
key: data-cache-0
261+
key: data-cache-tiny-0 # < 100 M, might as well combine
210262
paths:
211263
- ~/.mne
212-
- ~/mne_data/mTRF_1.5
264+
- ~/mne_data/MNE-kiloword-data # (28 M)
265+
- ~/mne_data/MNE-eegbci-data # (35 M)
266+
- ~/mne_data/MNE-misc-data # (39 M)
267+
- ~/mne_data/mTRF_1.5 # (56 M)
268+
- ~/mne_data/MNE-phantom-4DBTi # (77 M)
213269
- save_cache:
214-
key: data-cache-1
270+
key: data-cache-multimodal
215271
paths:
216-
- ~/mne_data/HF_SEF
217-
- ~/mne_data/MEGSIM
272+
- ~/mne_data/MNE-multimodal-data # (240 M)
218273
- save_cache:
219-
key: data-cache-2
274+
key: data-cache-limo
220275
paths:
221-
- ~/mne_data/MNE-brainstorm-data
222-
- ~/mne_data/MNE-eegbci-data
276+
- ~/mne_data/MNE-limo-data # (244 M)
223277
- save_cache:
224-
key: data-cache-3
278+
key: data-cache-bst-phantom-ctf
225279
paths:
226-
- ~/mne_data/MNE-fieldtrip_cmc-data
227-
- ~/mne_data/MNE-kiloword-data
280+
- ~/mne_data/MNE-brainstorm-data/bst_phantom_ctf # (177 M)
228281
- save_cache:
229-
key: data-cache-4
282+
key: data-cache-bst-bst-raw
230283
paths:
231-
- ~/mne_data/MNE-misc-data
232-
- ~/mne_data/MNE-multimodal-data
284+
- ~/mne_data/MNE-brainstorm-data/bst_raw # (830 M)
233285
- save_cache:
234-
key: data-cache-5
286+
key: data-cache-bst-phantom-elekta
235287
paths:
236-
- ~/mne_data/MNE-OPM-data
237-
- ~/mne_data/MNE-phantom-4DBTi
288+
- ~/mne_data/MNE-brainstorm-data/bst_phantom_elekta # (1.4 G)
238289
- save_cache:
239-
key: data-cache-6
290+
key: data-cache-bst-auditory
240291
paths:
241-
- ~/mne_data/MNE-sample-data
242-
- ~/mne_data/MNE-somato-data
292+
- ~/mne_data/MNE-brainstorm-data/bst_auditory # (2.9 G)
243293
- save_cache:
244-
key: data-cache-7
294+
key: data-cache-bst-resting
245295
paths:
246-
- ~/mne_data/MNE-spm-face
247-
- ~/mne_data/MNE-testing-ata
296+
- ~/mne_data/MNE-brainstorm-data/bst_resting # (4.5 G)
248297
- save_cache:
249-
key: data-cache-8
298+
key: data-cache-fieldtrip
250299
paths:
251-
- ~/mne_data/MNE-visual_92_categories-data
252-
- ~/mne_data/MNE-limo-data
300+
- ~/mne_data/MNE-fieldtrip_cmc-data # (699 M)
301+
- save_cache:
302+
key: data-cache-somato
303+
paths:
304+
- ~/mne_data/MNE-somato-data # (750 M)
305+
- save_cache:
306+
key: data-cache-hf-sef
307+
paths:
308+
- ~/mne_data/HF_SEF # (1.3 G)
309+
- save_cache:
310+
key: data-cache-opm
311+
paths:
312+
- ~/mne_data/MNE-OPM-data # (1.9 G)
313+
- save_cache:
314+
key: data-cache-sample
315+
paths:
316+
- ~/mne_data/MNE-sample-data # (3.2 G)
317+
- save_cache:
318+
key: data-cache-spm-face
319+
paths:
320+
- ~/mne_data/MNE-spm-face # (1.5 G)
321+
- save_cache:
322+
key: data-cache-testing
323+
paths:
324+
- ~/mne_data/MNE-testing-data # (2.5 G)
325+
- save_cache:
326+
key: data-cache-visual
327+
paths:
328+
- ~/mne_data/MNE-visual_92_categories-data # (6 G)
253329

254330

255331
linkcheck:

‎.github/workflows/compat_minimal.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,14 @@ jobs:
6464
- shell: bash -el {0}
6565
run: ./tools/github_actions_infos.sh
6666
name: 'Show infos'
67+
- shell: bash -el {0}
68+
run: ./tools/get_testing_version.sh
69+
name: 'Get testing version'
70+
- uses: actions/cache@v2
71+
with:
72+
key: ${{ env.TESTING_VERSION }}
73+
path: ~/mne_data
74+
name: 'Cache testing data'
6775
- shell: bash -el {0}
6876
run: ./tools/github_actions_download.sh
6977
name: 'Download testing data'

‎.github/workflows/compat_old.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,14 @@ jobs:
5757
- shell: bash -el {0}
5858
run: ./tools/github_actions_infos.sh
5959
name: 'Show infos'
60+
- shell: bash -el {0}
61+
run: ./tools/get_testing_version.sh
62+
name: 'Get testing version'
63+
- uses: actions/cache@v2
64+
with:
65+
key: ${{ env.TESTING_VERSION }}
66+
path: ~/mne_data
67+
name: 'Cache testing data'
6068
- shell: bash -el {0}
6169
run: ./tools/github_actions_download.sh
6270
name: 'Download testing data'

‎.github/workflows/linux_conda.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,14 @@ jobs:
5959
- shell: bash -el {0}
6060
run: ./tools/github_actions_infos.sh
6161
name: 'Show infos'
62+
- shell: bash -el {0}
63+
run: ./tools/get_testing_version.sh
64+
name: 'Get testing version'
65+
- uses: actions/cache@v2
66+
with:
67+
key: ${{ env.TESTING_VERSION }}
68+
path: ~/mne_data
69+
name: 'Cache testing data'
6270
- shell: bash -el {0}
6371
run: ./tools/github_actions_download.sh
6472
name: 'Download testing data'

‎.github/workflows/linux_pip.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,14 @@ jobs:
5757
- shell: bash -el {0}
5858
run: ./tools/github_actions_infos.sh
5959
name: 'Show infos'
60+
- shell: bash -el {0}
61+
run: ./tools/get_testing_version.sh
62+
name: 'Get testing version'
63+
- uses: actions/cache@v2
64+
with:
65+
key: ${{ env.TESTING_VERSION }}
66+
path: ~/mne_data
67+
name: 'Cache testing data'
6068
- shell: bash -el {0}
6169
run: ./tools/github_actions_download.sh
6270
name: 'Download testing data'

‎.github/workflows/macos_conda.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,14 @@ jobs:
5656
- shell: bash -el {0}
5757
run: ./tools/github_actions_infos.sh
5858
name: 'Show infos'
59+
- shell: bash -el {0}
60+
run: ./tools/get_testing_version.sh
61+
name: 'Get testing version'
62+
- uses: actions/cache@v2
63+
with:
64+
key: ${{ env.TESTING_VERSION }}
65+
path: ~/mne_data
66+
name: 'Cache testing data'
5967
- shell: bash -el {0}
6068
run: ./tools/github_actions_download.sh
6169
name: 'Download testing data'

0 commit comments

Comments
 (0)