Skip to content

Commit e97fcce

Browse files
Remi-GauRemi-Gau
authored
Merge pull request #197 from Remi-Gau/rm_data
[ENH] refactor removing date entity and allow to use bids fitler to only convert certain files from source to raw
2 parents 627ceef + 7bc55c0 commit e97fcce

File tree

4 files changed

+245
-146
lines changed

4 files changed

+245
-146
lines changed

‎.github/workflows/bids_validator.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,9 @@ name: BIDS validator
22

33
on:
44
push:
5-
branches:
6-
- master
5+
branches: ['master', 'main', 'dev']
76
pull_request:
8-
branches: ['master']
7+
branches: ['*']
98

109
env:
1110
OCTFLAGS: --no-gui --no-window-system --silent
@@ -15,7 +14,11 @@ jobs:
1514
runs-on: ubuntu-20.04
1615
steps:
1716

18-
- uses: actions/checkout@v2
17+
- uses: actions/setup-node@v3
18+
with:
19+
node-version: '14'
20+
21+
- uses: actions/checkout@v3
1922
with:
2023
submodules: true
2124
fetch-depth: 1
@@ -37,9 +40,10 @@ jobs:
3740
3841
- name: Install BIDS validator
3942
run: |
40-
sudo npm install -g bids-validator@1.9.0
43+
sudo npm install -g bids-validator
4144
4245
- name: Create dummy dataset and validate it
4346
run: |
4447
cd tests/manualTests
45-
octave $OCTFLAGS --eval "test_makeRawDataset" && bids-validator `pwd`/output/raw/ --ignoreNiftiHeaders
48+
octave $OCTFLAGS --eval "test_makeRawDataset"
49+
bids-validator `pwd`/output/raw/ --ignoreNiftiHeaders

‎src/convertSourceToRaw.m‎

Lines changed: 69 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
1-
function convertSourceToRaw(cfg)
1+
function convertSourceToRaw(varargin)
22
%
33
% Function attempts to convert a source dataset created with CPP_BIDS into a valid
44
% BIDS data set.
55
%
66
%
77
% USAGE::
88
%
9-
% convertSourceToRaw(cfg)
9+
% convertSourceToRaw(cfg, 'filter', filter)
1010
%
1111
% :param cfg: cfg structure is needed only for providing the path in ``cfg.dir.output``.
1212
% :type cfg: structure
1313
%
14+
% :param filter: bids.query filter to only convert a subset of files.
15+
% :type filter: structure
16+
%
1417
% :output:
1518
% - :creates: a dummy README and CHANGE file
1619
% - :copies: ``source`` directory to ``raw`` directory
@@ -19,39 +22,80 @@ function convertSourceToRaw(cfg)
1922
%
2023
% (C) Copyright 2020 CPP_BIDS developers
2124

25+
args = inputParser;
26+
27+
default_filter = struct([]);
28+
29+
args.addRequired('cfg', @isstruct);
30+
args.addParameter('filter', default_filter, @isstruct);
31+
32+
args.parse(varargin{:});
33+
34+
cfg = args.Results.cfg;
35+
filter = args.Results.filter;
36+
2237
sourceDir = fullfile(cfg.dir.output, 'source');
2338
rawDir = fullfile(cfg.dir.output, 'raw');
2439

25-
% add dummy README and CHANGE file
26-
templateFolder = fullfile(fileparts(mfilename('fullpath')), '..', 'templates');
40+
% back up description to not overwrite
41+
% TODO bids malab should be smart enought to not do that
42+
isFile = @(x) exist(x, 'file');
43+
if isFile(fullfile(rawDir, 'dataset_description.json'))
44+
copyfile(fullfile(rawDir, 'dataset_description.json'), ...
45+
fullfile(rawDir, 'dataset_description_bu.json'));
46+
end
2747

28-
copyfile(fullfile(templateFolder, 'README'), ...
29-
sourceDir);
30-
copyfile(fullfile(templateFolder, 'CHANGES'), ...
31-
sourceDir);
32-
copyfile(fullfile(templateFolder, '.bidsignore'), ...
33-
sourceDir);
48+
% trick use bids matlab copy dataset function
49+
bids.copy_to_derivative(sourceDir, ...
50+
'pipeline_name', '.', ...
51+
'out_path', rawDir, ...
52+
'filter', filter, ...
53+
'unzip', false, ...
54+
'force', true, ...
55+
'skip_dep', false, ...
56+
'use_schema', false, ...
57+
'verbose', true);
3458

35-
copyfile(sourceDir, rawDir);
59+
if isFile(fullfile(rawDir, 'dataset_description_bu.json'))
60+
copyfile(fullfile(rawDir, 'dataset_description_bu.json'), ...
61+
fullfile(rawDir, 'dataset_description.json'));
62+
delete(fullfile(rawDir, 'dataset_description_bu.json'));
63+
else
64+
% clean up description
65+
description = bids.util.jsondecode(fullfile(rawDir, 'dataset_description.json'));
66+
description.BIDSVersion = '1.7.0';
67+
description.Name = 'FIXME';
68+
description.DatasetType = 'raw';
69+
description = rmfield(description, 'GeneratedBy');
70+
description = rmfield(description, 'SourceDatasets');
71+
bids.util.jsonencode(fullfile(rawDir, 'dataset_description.json'), description);
72+
end
3673

37-
BIDS = bids.layout(rawDir, 'use_schema', false);
74+
removeDateEntity(rawDir, 'filter', filter);
3875

39-
data = bids.query(BIDS, 'data');
40-
metadata = bids.query(BIDS, 'metadata');
76+
gunzipTimeSeries(rawDir);
4177

42-
for i = 1:size(data, 1)
43-
bf = bids.File(data{i});
44-
if isfield(bf.entities, 'date')
45-
% TODO probably JSON renaming should be passed to bids-matlab
46-
sourceJson = fullfile(fileparts(bf.path), bf.json_filename);
47-
bf.entities.date = '';
48-
bf.rename('dry_run', false, 'force', true);
49-
bids.util.jsonencode(fullfile(fileparts(bf.path), bf.json_filename), metadata{i});
50-
delete(sourceJson);
51-
end
78+
% add dummy README and CHANGE file
79+
templateFolder = fullfile(fileparts(mfilename('fullpath')), '..', 'templates');
80+
81+
if ~isFile(fullfile(rawDir, 'README'))
82+
copyfile(fullfile(templateFolder, 'README'), ...
83+
rawDir);
84+
end
85+
if ~isFile(fullfile(rawDir, 'CHANGES'))
86+
copyfile(fullfile(templateFolder, 'CHANGES'), ...
87+
rawDir);
5288
end
89+
if ~isFile(fullfile(rawDir, '.bidsignore'))
90+
copyfile(fullfile(templateFolder, '.bidsignore'), ...
91+
rawDir);
92+
end
93+
94+
end
5395

54-
BIDS = bids.layout(rawDir, 'use_schema', false);
96+
function gunzipTimeSeries(pathToDataSet)
97+
98+
BIDS = bids.layout(pathToDataSet, 'use_schema', false);
5599
data = bids.query(BIDS, 'data', 'suffix', {'stim', 'physio' }, 'ext', '.tsv');
56100

57101
for i = 1:size(data, 1)
@@ -60,5 +104,4 @@ function convertSourceToRaw(cfg)
60104
delete(data{i});
61105
end
62106
end
63-
64107
end

‎src/utils/removeDateEntity.m‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
function removeDateEntity(varargin)
2+
%
3+
% Removes the date entity from all the files in a BIDS data set
4+
%
5+
%
6+
% USAGE::
7+
%
8+
% removeDateEntity(pathToDataSet, 'filter', filter)
9+
%
10+
%
11+
% (C) Copyright 2022 CPP_BIDS developers
12+
13+
args = inputParser;
14+
15+
default_filter = struct([]);
16+
17+
args.addRequired('pathToDataSet', @isdir);
18+
args.addParameter('filter', default_filter, @isstruct);
19+
20+
args.parse(varargin{:});
21+
22+
pathToDataSet = args.Results.pathToDataSet;
23+
filter = args.Results.filter;
24+
25+
BIDS = bids.layout(pathToDataSet, 'use_schema', false);
26+
27+
filter.date = '[0-9]+';
28+
data = bids.query(BIDS, 'data', filter);
29+
metadata = bids.query(BIDS, 'metadata', filter);
30+
31+
for i = 1:size(data, 1)
32+
bf = bids.File(data{i});
33+
% TODO probably JSON renaming should be passed to bids-matlab
34+
sourceJson = fullfile(fileparts(bf.path), bf.json_filename);
35+
bf.entities.date = '';
36+
bf.rename('dry_run', false, 'force', true);
37+
bids.util.jsonencode(fullfile(fileparts(bf.path), bf.json_filename), metadata{i});
38+
delete(sourceJson);
39+
end
40+
41+
end

0 commit comments

Comments
 (0)