Skip to content

Commit c25dd0f

Browse files
fangqfangq
committed
support blosc2 codes in save and load data, upgrade jsave/jload
1 parent b23181a commit c25dd0f

File tree

7 files changed

+73
-59
lines changed

7 files changed

+73
-59
lines changed

‎jdataencode.m

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -314,16 +314,20 @@
314314
end
315315

316316
if(~isempty(zipmethod) && numel(item)>minsize)
317-
compfun=str2func([regexprep(zipmethod,'blosc2[a-z0-9]+','blosc2') 'encode']);
317+
encodeparam={};
318+
if(~isempty(regexp(zipmethod,'^blosc2', 'once')))
319+
compfun=@blosc2encode;
320+
encodeparam={zipmethod, 'nthread', jsonopt('nthread',1,varargin{1}),...
321+
'shuffle', jsonopt('shuffle',1,varargin{1}), ...
322+
'typesize', jsonopt('typesize',length(typecast(item(1),'uint8')),varargin{1})};
323+
else
324+
compfun=str2func([zipmethod 'encode']);
325+
end
318326
newitem.(N('_ArrayZipType_'))=lower(zipmethod);
319327
if(~isfield(newitem,N('_ArrayZipSize_')))
320328
newitem.(N('_ArrayZipSize_'))=size(newitem.(N('_ArrayData_')));
321329
end
322-
if(strcmp(zipmethod,'blosc2'))
323-
newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'), zipmethod);
324-
else
325-
newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'));
326-
end
330+
newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'), encodeparam{:});
327331
newitem=rmfield(newitem,N('_ArrayData_'));
328332
if(varargin{1}.base64)
329333
newitem.(N('_ArrayZipData_'))=char(base64encode(newitem.(N('_ArrayZipData_'))));

‎jload.m

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -60,15 +60,6 @@
6060

6161
ws=jsonopt('ws','caller',opt);
6262

63-
loadfun=@loadbj;
64-
if(regexp(filename,'\.[jJ][sS][oO][nN]$'))
65-
loadfun=@loadjson;
66-
elseif(regexp(filename,'\.[jJ][dD][tT]$'))
67-
loadfun=@loadjson;
68-
elseif(regexp(filename,'\.[mM][sS][gG][pP][kK]$'))
69-
loadfun=@loadmsgpack;
70-
end
71-
7263
if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
7364
jsonstr=fileread(filename);
7465
pos=regexp(jsonstr,'}\n\n\n{"WorkspaceData":','once');
@@ -78,9 +69,9 @@
7869
header=jsondecode(jsonstr(1:pos+1));
7970
else
8071
try
81-
header=loadfun(filename,'ObjectID',1, 'MaxBuffer', 65536, varargin{:});
72+
header=loadjd(filename,'ObjectID',1, 'MaxBuffer', 65536, varargin{:});
8273
catch
83-
header=loadfun(filename,'ObjectID',1, varargin{:});
74+
header=loadjd(filename,'ObjectID',1, varargin{:});
8475
end
8576
end
8677

@@ -97,7 +88,7 @@
9788
if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
9889
body=jdatadecode(jsondecode(jsonstr(pos+4:end)));
9990
else
100-
body=loadfun(filename,'ObjectID',2, varargin{:});
91+
body=loadjd(filename,'ObjectID',2, varargin{:});
10192
end
10293

10394
if(nargout==0)

‎jsave.m

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,22 @@
4949
end
5050

5151
opt=varargin2struct(varargin{:});
52+
if(~isfield(opt,'nthread'))
53+
opt.nthread=4;
54+
end
55+
if(~isfield(opt,'compression'))
56+
if(exist('zipmat'))
57+
opt.compression='blosc2zstd';
58+
else
59+
opt.compression='zlib';
60+
end
61+
end
62+
if(~isfield(opt,'shuffle'))
63+
opt.shuffle=1;
64+
end
65+
if(~isfield(opt,'typesize'))
66+
opt.typesize=4;
67+
end
5268

5369
ws=jsonopt('ws','caller',opt);
5470

@@ -90,22 +106,13 @@
90106
body.(varlist{i})=evalin(ws,varlist{i});
91107
end
92108

93-
savefun=@savebj;
94-
loadfun=@loadbj;
95-
if(regexp(filename,'\.[jJ][sS][oO][nN]$'))
96-
savefun=@savejson;
97-
loadfun=@loadjson;
98-
elseif(regexp(filename,'\.[jJ][dD][tT]$'))
99-
savefun=@savejson;
100-
loadfun=@loadjson;
101-
elseif(regexp(filename,'\.[mM][sS][gG][pP][kK]$'))
102-
savefun=@savemsgpack;
103-
end
104-
105109
if(nargout==1)
106110
varargout{1}=header;
107111
end
108112

113+
defaultopt={'compression',opt.compression,'nthread',opt.nthread,...
114+
'shuffle',opt.shuffle,'typesize',opt.typesize,'keeptype',1,'array2struct',1};
115+
109116
if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
110117
if(isempty(regexp(filename,'\.[jJ][sS][oO][nN]$', 'once')))
111118
filename=regexprep(filename,'\.[a-zA-Z]+$','.jdt');
@@ -115,8 +122,7 @@
115122
clear output;
116123

117124
output.WorkspaceData=jdataencode(body,'AnnotateArray',1,'base64',1,...
118-
'Compression','zlib','UseArrayZipSize',1,'MapAsStruct',1,...
119-
'prefix','x',varargin{:});
125+
'UseArrayZipSize',1,'MapAsStruct',1,'prefix','x',defaultopt{:},varargin{:});
120126
bodyjson=jsonencode(output);
121127
clear output;
122128

@@ -126,15 +132,15 @@
126132
fwrite(fid,bodyjson);
127133
fclose(fid);
128134
elseif(jsonopt('usemmap',0,opt)==1)
129-
bodyjson=savefun('WorkspaceData',body,...
130-
'compression','zlib','keeptype',1,'array2struct',1,varargin{:});
131-
header.(encodevarname('_MMap_'))=loadfun(bodyjson,'mmaponly',1,varargin{:});
132-
savefun('WorkspaceHeader',header,'filename',filename,varargin{:});
135+
bodyjson=savejd('WorkspaceData',body,...
136+
defaultopt{:},varargin{:});
137+
header.(encodevarname('_MMap_'))=loadjd(bodyjson,'mmaponly',1,varargin{:});
138+
savejd('WorkspaceHeader',header,'filename',filename,varargin{:});
133139
fid=fopen(filename,'ab+');
134140
fwrite(fid,bodyjson);
135141
fclose(fid);
136142
else
137-
savefun('WorkspaceHeader',header,'filename',filename,varargin{:});
138-
savefun('WorkspaceData',body,'filename',filename,'append',1,...
139-
'compression','zlib','keeptype',1,'array2struct',1,varargin{:});
143+
savejd('WorkspaceHeader',header,'filename',filename,varargin{:});
144+
savejd('WorkspaceData',body,'filename',filename,'append',1,...
145+
defaultopt{:},varargin{:});
140146
end

‎loadjd.m

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
% input:
1414
% inputfile: the input hierarchical container data file, supporting:
1515
% *.json,.jnii,.jdt,.jmsh,.jnirs,.jbids: JSON/JData based data files, see https://neurojson.org/jdata/draft2
16-
% *.bjd,.bnii,.jdb,.bmsh,.bnirs: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
16+
% *.bjd,.bnii,.jdb,.bmsh,.bnirs,.pmat: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
1717
% *.ubj: UBJSON-encoded files, see http://ubjson.org
1818
% *.msgpack: MessagePack-encoded files, see http://msgpack.org
1919
% *.h5,.hdf5,.snirf: HDF5 files, see https://www.hdfgroup.org/
@@ -45,7 +45,7 @@
4545

4646
if(regexpi(filename,'\.json$|\.jnii$|\.jdt$|\.jdat$|\.jmsh$|\.jnirs|\.jbids$'))
4747
[varargout{1:nargout}]=loadjson(filename,varargin{:});
48-
elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$'))
48+
elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$|\.pmat'))
4949
[varargout{1:nargout}]=loadbj(filename,varargin{:});
5050
elseif(regexpi(filename,'\.ubj$'))
5151
[varargout{1:nargout}]=loadubjson(filename,varargin{:});
@@ -57,5 +57,5 @@
5757
end
5858
[varargout{1:nargout}]=loadh5(filename,varargin{:});
5959
else
60-
error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf');
60+
error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf,.pmat');
6161
end

‎savebj.m

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -656,8 +656,14 @@
656656
cid=I_(uint32(max(size(fulldata))),varargin{:});
657657
txt=[txt, N_('_ArrayZipSize_',opt),I_a(size(fulldata),cid(1),varargin{:})];
658658
txt=[txt, N_('_ArrayZipType_',opt),S_(dozip,opt)];
659-
compfun=str2func([dozip 'encode']);
660-
txt=[txt,N_('_ArrayZipData_',opt), I_a(compfun(typecast(fulldata(:),'uint8')),Imarker(1),varargin{:})];
659+
encodeparam={};
660+
if(~isempty(regexp(dozip,'^blosc2', 'once')))
661+
compfun=@blosc2encode;
662+
encodeparam={dozip, 'nthread', jsonopt('nthread',1,opt), 'shuffle', jsonopt('shuffle',1,opt), 'typesize', jsonopt('typesize',length(typecast(fulldata(1),'uint8')),opt)};
663+
else
664+
compfun=str2func([dozip 'encode']);
665+
end
666+
txt=[txt,N_('_ArrayZipData_',opt), I_a(compfun(typecast(fulldata(:),'uint8'),encodeparam{:}),Imarker(1),varargin{:})];
661667
childcount=childcount+3;
662668
else
663669
if(ismsgpack)

‎savejd.m

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
% class instance).
2222
% outputfile: the output file name to the hierarchical container file
2323
% *.json,.jnii,.jdt,.jmsh,.jnirs,.jbids: JSON/JData based data files, see https://neurojson.org/jdata/draft2
24-
% *.bjd,.bnii,.jdb,.bmsh,.bnirs: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
24+
% *.bjd,.bnii,.jdb,.bmsh,.bnirs,.pmat: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
2525
% *.ubj: UBJSON-encoded files, see http://ubjson.org
2626
% *.msgpack: MessagePack-encoded files, see http://msgpack.org
2727
% *.h5,.hdf5,.snirf: HDF5 files, see https://www.hdfgroup.org/
@@ -66,7 +66,7 @@
6666
end
6767
if(regexpi(filename,'\.json$|\.jnii$|\.jdt$|\.jdat$|\.jmsh$|\.jnirs$|\.jbids$'))
6868
[varargout{1:nargout}]=savejson(varargin{:});
69-
elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$'))
69+
elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$|\.pmat'))
7070
[varargout{1:nargout}]=savebj(varargin{:});
7171
elseif(regexpi(filename,'\.ubj$'))
7272
[varargout{1:nargout}]=saveubjson(varargin{:});
@@ -85,5 +85,5 @@
8585
end
8686
[varargout{1:nargout}]=saveh5(varargin{2},filename, opt);
8787
else
88-
error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf');
88+
error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf,.pmat');
8989
end

‎savejson.m

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -565,35 +565,36 @@
565565
if (~isnumeric(item) && ~islogical(item) && ~ischar(item))
566566
error('input is not an array');
567567
end
568-
ws = varargin{1}.whitespaces_;
568+
opt=varargin{1};
569+
ws = opt.whitespaces_;
569570
padding1 = repmat(ws.tab, 1, level);
570571
padding0 = repmat(ws.tab, 1, level + 1);
571572
nl = ws.newline;
572573
sep = ws.sep;
573574

574-
dozip = varargin{1}.compression;
575-
zipsize = varargin{1}.compressarraysize;
576-
format = varargin{1}.formatversion;
577-
isnest = varargin{1}.nestarray;
575+
dozip = opt.compression;
576+
zipsize = opt.compressarraysize;
577+
format = opt.formatversion;
578+
isnest = opt.nestarray;
578579

579-
if (~varargin{1}.nosubstruct_ && (((isnest == 0) && length(size(item)) > 2) || issparse(item) || ~isreal(item) || ...
580-
(isempty(item) && any(size(item))) || varargin{1}.arraytostruct || (~isempty(dozip) && numel(item) > zipsize)))
580+
if (~opt.nosubstruct_ && (((isnest == 0) && length(size(item)) > 2) || issparse(item) || ~isreal(item) || ...
581+
(isempty(item) && any(size(item))) || opt.arraytostruct || (~isempty(dozip) && numel(item) > zipsize)))
581582
if (isempty(name))
582583
txt = sprintf('%s{%s%s"_ArrayType_":"%s",%s%s"_ArraySize_":%s,%s', ...
583584
padding1, nl, padding0, class(item), nl, padding0, regexprep(mat2str(size(item)), '\s+', ','), nl);
584585
else
585586
txt = sprintf('%s"%s":{%s%s"_ArrayType_":"%s",%s%s"_ArraySize_":%s,%s', ...
586-
padding1, decodevarname(name, varargin{1}.unpackhex), nl, padding0, class(item), nl, padding0, regexprep(mat2str(size(item)), '\s+', ','), nl);
587+
padding1, decodevarname(name, opt.unpackhex), nl, padding0, class(item), nl, padding0, regexprep(mat2str(size(item)), '\s+', ','), nl);
587588
end
588589
else
589590
numtxt = matdata2json(item, level + 1, varargin{:});
590591
if (isempty(name))
591592
txt = sprintf('%s%s', padding1, numtxt);
592593
else
593594
if (numel(item) == 1 && varargin{1}.singletarray == 0)
594-
txt = sprintf('%s"%s":%s', padding1, decodevarname(name, varargin{1}.unpackhex), numtxt);
595+
txt = sprintf('%s"%s":%s', padding1, decodevarname(name, opt.unpackhex), numtxt);
595596
else
596-
txt = sprintf('%s"%s":%s', padding1, decodevarname(name, varargin{1}.unpackhex), numtxt);
597+
txt = sprintf('%s"%s":%s', padding1, decodevarname(name, opt.unpackhex), numtxt);
597598
end
598599
end
599600
return
@@ -659,8 +660,14 @@
659660
end
660661
txt = sprintf(dataformat, txt, padding0, '"_ArrayZipSize_":', regexprep(mat2str(size(fulldata)), '\s+', ','), sep);
661662
txt = sprintf(dataformat, txt, padding0, '"_ArrayZipType_":"', dozip, ['"' sep]);
662-
compfun = str2func([dozip 'encode']);
663-
txt = sprintf(dataformat, txt, padding0, '"_ArrayZipData_":"', char(base64encode(compfun(typecast(fulldata(:), 'uint8')))), ['"' nl]);
663+
encodeparam={};
664+
if(~isempty(regexp(dozip,'^blosc2', 'once')))
665+
compfun=@blosc2encode;
666+
encodeparam={dozip, 'nthread', jsonopt('nthread',1,opt), 'shuffle', jsonopt('shuffle',1,opt), 'typesize', jsonopt('typesize',length(typecast(fulldata(1),'uint8')),opt)};
667+
else
668+
compfun=str2func([dozip 'encode']);
669+
end
670+
txt = sprintf(dataformat, txt, padding0, '"_ArrayZipData_":"', char(base64encode(compfun(typecast(fulldata(:), 'uint8'),encodeparam{:}))), ['"' nl]);
664671
else
665672
if (isreal(item))
666673
txt = sprintf(dataformat, txt, padding0, '"_ArrayData_":', ...

0 commit comments

Comments
 (0)