Skip to content

Commit

Permalink
support blosc2 codes in save and load data, upgrade jsave/jload
Browse files Browse the repository at this point in the history
  • Loading branch information
@fangq
fangq committed Jun 27, 2023
1 parent b23181a commit c25dd0f
Show file tree
Hide file tree
Showing 7 changed files with 73 additions and 59 deletions.
16 changes: 10 additions & 6 deletions jdataencode.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,16 +314,20 @@
end

if(~isempty(zipmethod) && numel(item)>minsize)
compfun=str2func([regexprep(zipmethod,'blosc2[a-z0-9]+','blosc2') 'encode']);
encodeparam={};
if(~isempty(regexp(zipmethod,'^blosc2', 'once')))
compfun=@blosc2encode;
encodeparam={zipmethod, 'nthread', jsonopt('nthread',1,varargin{1}),...
'shuffle', jsonopt('shuffle',1,varargin{1}), ...
'typesize', jsonopt('typesize',length(typecast(item(1),'uint8')),varargin{1})};
else
compfun=str2func([zipmethod 'encode']);
end
newitem.(N('_ArrayZipType_'))=lower(zipmethod);
if(~isfield(newitem,N('_ArrayZipSize_')))
newitem.(N('_ArrayZipSize_'))=size(newitem.(N('_ArrayData_')));
end
if(strcmp(zipmethod,'blosc2'))
newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'), zipmethod);
else
newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'));
end
newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'), encodeparam{:});
newitem=rmfield(newitem,N('_ArrayData_'));
if(varargin{1}.base64)
newitem.(N('_ArrayZipData_'))=char(base64encode(newitem.(N('_ArrayZipData_'))));
Expand Down
15 changes: 3 additions & 12 deletions jload.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,6 @@

ws=jsonopt('ws','caller',opt);

loadfun=@loadbj;
if(regexp(filename,'\.[jJ][sS][oO][nN]$'))
loadfun=@loadjson;
elseif(regexp(filename,'\.[jJ][dD][tT]$'))
loadfun=@loadjson;
elseif(regexp(filename,'\.[mM][sS][gG][pP][kK]$'))
loadfun=@loadmsgpack;
end

if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
jsonstr=fileread(filename);
pos=regexp(jsonstr,'}\n\n\n{"WorkspaceData":','once');
Expand All @@ -78,9 +69,9 @@
header=jsondecode(jsonstr(1:pos+1));
else
try
header=loadfun(filename,'ObjectID',1, 'MaxBuffer', 65536, varargin{:});
header=loadjd(filename,'ObjectID',1, 'MaxBuffer', 65536, varargin{:});
catch
header=loadfun(filename,'ObjectID',1, varargin{:});
header=loadjd(filename,'ObjectID',1, varargin{:});
end
end

Expand All @@ -97,7 +88,7 @@
if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
body=jdatadecode(jsondecode(jsonstr(pos+4:end)));
else
body=loadfun(filename,'ObjectID',2, varargin{:});
body=loadjd(filename,'ObjectID',2, varargin{:});
end

if(nargout==0)
Expand Down
48 changes: 27 additions & 21 deletions jsave.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,22 @@
end

opt=varargin2struct(varargin{:});
if(~isfield(opt,'nthread'))
opt.nthread=4;
end
if(~isfield(opt,'compression'))
if(exist('zipmat'))
opt.compression='blosc2zstd';
else
opt.compression='zlib';
end
end
if(~isfield(opt,'shuffle'))
opt.shuffle=1;
end
if(~isfield(opt,'typesize'))
opt.typesize=4;
end

ws=jsonopt('ws','caller',opt);

Expand Down Expand Up @@ -90,22 +106,13 @@
body.(varlist{i})=evalin(ws,varlist{i});
end

savefun=@savebj;
loadfun=@loadbj;
if(regexp(filename,'\.[jJ][sS][oO][nN]$'))
savefun=@savejson;
loadfun=@loadjson;
elseif(regexp(filename,'\.[jJ][dD][tT]$'))
savefun=@savejson;
loadfun=@loadjson;
elseif(regexp(filename,'\.[mM][sS][gG][pP][kK]$'))
savefun=@savemsgpack;
end

if(nargout==1)
varargout{1}=header;
end

defaultopt={'compression',opt.compression,'nthread',opt.nthread,...
'shuffle',opt.shuffle,'typesize',opt.typesize,'keeptype',1,'array2struct',1};

if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
if(isempty(regexp(filename,'\.[jJ][sS][oO][nN]$', 'once')))
filename=regexprep(filename,'\.[a-zA-Z]+$','.jdt');
Expand All @@ -115,8 +122,7 @@
clear output;

output.WorkspaceData=jdataencode(body,'AnnotateArray',1,'base64',1,...
'Compression','zlib','UseArrayZipSize',1,'MapAsStruct',1,...
'prefix','x',varargin{:});
'UseArrayZipSize',1,'MapAsStruct',1,'prefix','x',defaultopt{:},varargin{:});
bodyjson=jsonencode(output);
clear output;

Expand All @@ -126,15 +132,15 @@
fwrite(fid,bodyjson);
fclose(fid);
elseif(jsonopt('usemmap',0,opt)==1)
bodyjson=savefun('WorkspaceData',body,...
'compression','zlib','keeptype',1,'array2struct',1,varargin{:});
header.(encodevarname('_MMap_'))=loadfun(bodyjson,'mmaponly',1,varargin{:});
savefun('WorkspaceHeader',header,'filename',filename,varargin{:});
bodyjson=savejd('WorkspaceData',body,...
defaultopt{:},varargin{:});
header.(encodevarname('_MMap_'))=loadjd(bodyjson,'mmaponly',1,varargin{:});
savejd('WorkspaceHeader',header,'filename',filename,varargin{:});
fid=fopen(filename,'ab+');
fwrite(fid,bodyjson);
fclose(fid);
else
savefun('WorkspaceHeader',header,'filename',filename,varargin{:});
savefun('WorkspaceData',body,'filename',filename,'append',1,...
'compression','zlib','keeptype',1,'array2struct',1,varargin{:});
savejd('WorkspaceHeader',header,'filename',filename,varargin{:});
savejd('WorkspaceData',body,'filename',filename,'append',1,...
defaultopt{:},varargin{:});
end
6 changes: 3 additions & 3 deletions loadjd.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
% input:
% inputfile: the input hierarchical container data file, supporting:
% *.json,.jnii,.jdt,.jmsh,.jnirs,.jbids: JSON/JData based data files, see https://neurojson.org/jdata/draft2
% *.bjd,.bnii,.jdb,.bmsh,.bnirs: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
% *.bjd,.bnii,.jdb,.bmsh,.bnirs,.pmat: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
% *.ubj: UBJSON-encoded files, see http://ubjson.org
% *.msgpack: MessagePack-encoded files, see http://msgpack.org
% *.h5,.hdf5,.snirf: HDF5 files, see https://www.hdfgroup.org/
Expand Down Expand Up @@ -45,7 +45,7 @@

if(regexpi(filename,'\.json$|\.jnii$|\.jdt$|\.jdat$|\.jmsh$|\.jnirs|\.jbids$'))
[varargout{1:nargout}]=loadjson(filename,varargin{:});
elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$'))
elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$|\.pmat'))
[varargout{1:nargout}]=loadbj(filename,varargin{:});
elseif(regexpi(filename,'\.ubj$'))
[varargout{1:nargout}]=loadubjson(filename,varargin{:});
Expand All @@ -57,5 +57,5 @@
end
[varargout{1:nargout}]=loadh5(filename,varargin{:});
else
error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf');
error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf,.pmat');
end
10 changes: 8 additions & 2 deletions savebj.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -656,8 +656,14 @@
cid=I_(uint32(max(size(fulldata))),varargin{:});
txt=[txt, N_('_ArrayZipSize_',opt),I_a(size(fulldata),cid(1),varargin{:})];
txt=[txt, N_('_ArrayZipType_',opt),S_(dozip,opt)];
compfun=str2func([dozip 'encode']);
txt=[txt,N_('_ArrayZipData_',opt), I_a(compfun(typecast(fulldata(:),'uint8')),Imarker(1),varargin{:})];
encodeparam={};
if(~isempty(regexp(dozip,'^blosc2', 'once')))
compfun=@blosc2encode;
encodeparam={dozip, 'nthread', jsonopt('nthread',1,opt), 'shuffle', jsonopt('shuffle',1,opt), 'typesize', jsonopt('typesize',length(typecast(fulldata(1),'uint8')),opt)};
else
compfun=str2func([dozip 'encode']);
end
txt=[txt,N_('_ArrayZipData_',opt), I_a(compfun(typecast(fulldata(:),'uint8'),encodeparam{:}),Imarker(1),varargin{:})];
childcount=childcount+3;
else
if(ismsgpack)
Expand Down
6 changes: 3 additions & 3 deletions savejd.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
% class instance).
% outputfile: the output file name to the hierarchical container file
% *.json,.jnii,.jdt,.jmsh,.jnirs,.jbids: JSON/JData based data files, see https://neurojson.org/jdata/draft2
% *.bjd,.bnii,.jdb,.bmsh,.bnirs: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
% *.bjd,.bnii,.jdb,.bmsh,.bnirs,.pmat: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
% *.ubj: UBJSON-encoded files, see http://ubjson.org
% *.msgpack: MessagePack-encoded files, see http://msgpack.org
% *.h5,.hdf5,.snirf: HDF5 files, see https://www.hdfgroup.org/
Expand Down Expand Up @@ -66,7 +66,7 @@
end
if(regexpi(filename,'\.json$|\.jnii$|\.jdt$|\.jdat$|\.jmsh$|\.jnirs$|\.jbids$'))
[varargout{1:nargout}]=savejson(varargin{:});
elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$'))
elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$|\.pmat'))
[varargout{1:nargout}]=savebj(varargin{:});
elseif(regexpi(filename,'\.ubj$'))
[varargout{1:nargout}]=saveubjson(varargin{:});
Expand All @@ -85,5 +85,5 @@
end
[varargout{1:nargout}]=saveh5(varargin{2},filename, opt);
else
error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf');
error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf,.pmat');
end
31 changes: 19 additions & 12 deletions savejson.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -565,35 +565,36 @@
if (~isnumeric(item) && ~islogical(item) && ~ischar(item))
error('input is not an array');
end
ws = varargin{1}.whitespaces_;
opt=varargin{1};
ws = opt.whitespaces_;
padding1 = repmat(ws.tab, 1, level);
padding0 = repmat(ws.tab, 1, level + 1);
nl = ws.newline;
sep = ws.sep;

dozip = varargin{1}.compression;
zipsize = varargin{1}.compressarraysize;
format = varargin{1}.formatversion;
isnest = varargin{1}.nestarray;
dozip = opt.compression;
zipsize = opt.compressarraysize;
format = opt.formatversion;
isnest = opt.nestarray;

if (~varargin{1}.nosubstruct_ && (((isnest == 0) && length(size(item)) > 2) || issparse(item) || ~isreal(item) || ...
(isempty(item) && any(size(item))) || varargin{1}.arraytostruct || (~isempty(dozip) && numel(item) > zipsize)))
if (~opt.nosubstruct_ && (((isnest == 0) && length(size(item)) > 2) || issparse(item) || ~isreal(item) || ...
(isempty(item) && any(size(item))) || opt.arraytostruct || (~isempty(dozip) && numel(item) > zipsize)))
if (isempty(name))
txt = sprintf('%s{%s%s"_ArrayType_":"%s",%s%s"_ArraySize_":%s,%s', ...
padding1, nl, padding0, class(item), nl, padding0, regexprep(mat2str(size(item)), '\s+', ','), nl);
else
txt = sprintf('%s"%s":{%s%s"_ArrayType_":"%s",%s%s"_ArraySize_":%s,%s', ...
padding1, decodevarname(name, varargin{1}.unpackhex), nl, padding0, class(item), nl, padding0, regexprep(mat2str(size(item)), '\s+', ','), nl);
padding1, decodevarname(name, opt.unpackhex), nl, padding0, class(item), nl, padding0, regexprep(mat2str(size(item)), '\s+', ','), nl);
end
else
numtxt = matdata2json(item, level + 1, varargin{:});
if (isempty(name))
txt = sprintf('%s%s', padding1, numtxt);
else
if (numel(item) == 1 && varargin{1}.singletarray == 0)
txt = sprintf('%s"%s":%s', padding1, decodevarname(name, varargin{1}.unpackhex), numtxt);
txt = sprintf('%s"%s":%s', padding1, decodevarname(name, opt.unpackhex), numtxt);
else
txt = sprintf('%s"%s":%s', padding1, decodevarname(name, varargin{1}.unpackhex), numtxt);
txt = sprintf('%s"%s":%s', padding1, decodevarname(name, opt.unpackhex), numtxt);
end
end
return
Expand Down Expand Up @@ -659,8 +660,14 @@
end
txt = sprintf(dataformat, txt, padding0, '"_ArrayZipSize_":', regexprep(mat2str(size(fulldata)), '\s+', ','), sep);
txt = sprintf(dataformat, txt, padding0, '"_ArrayZipType_":"', dozip, ['"' sep]);
compfun = str2func([dozip 'encode']);
txt = sprintf(dataformat, txt, padding0, '"_ArrayZipData_":"', char(base64encode(compfun(typecast(fulldata(:), 'uint8')))), ['"' nl]);
encodeparam={};
if(~isempty(regexp(dozip,'^blosc2', 'once')))
compfun=@blosc2encode;
encodeparam={dozip, 'nthread', jsonopt('nthread',1,opt), 'shuffle', jsonopt('shuffle',1,opt), 'typesize', jsonopt('typesize',length(typecast(fulldata(1),'uint8')),opt)};
else
compfun=str2func([dozip 'encode']);
end
txt = sprintf(dataformat, txt, padding0, '"_ArrayZipData_":"', char(base64encode(compfun(typecast(fulldata(:), 'uint8'),encodeparam{:}))), ['"' nl]);
else
if (isreal(item))
txt = sprintf(dataformat, txt, padding0, '"_ArrayData_":', ...
Expand Down

0 comments on commit c25dd0f

Please sign in to comment.