support blosc2 codes in save and load data, upgrade jsave/jload

Author: fangq

jdataencode.m

@@ -314,16 +314,20 @@
 end
 
 if(~isempty(zipmethod) && numel(item)>minsize)
-    compfun=str2func([regexprep(zipmethod,'blosc2[a-z0-9]+','blosc2') 'encode']);
+    encodeparam={};
+    if(~isempty(regexp(zipmethod,'^blosc2', 'once')))
+        compfun=@blosc2encode;
+        encodeparam={zipmethod, 'nthread', jsonopt('nthread',1,varargin{1}),...
+            'shuffle', jsonopt('shuffle',1,varargin{1}), ...
+            'typesize', jsonopt('typesize',length(typecast(item(1),'uint8')),varargin{1})};
+    else
+        compfun=str2func([zipmethod 'encode']);
+    end
     newitem.(N('_ArrayZipType_'))=lower(zipmethod);
     if(~isfield(newitem,N('_ArrayZipSize_')))
         newitem.(N('_ArrayZipSize_'))=size(newitem.(N('_ArrayData_')));
     end
-    if(strcmp(zipmethod,'blosc2'))
-        newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'), zipmethod);
-    else
-        newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'));
-    end
+    newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'), encodeparam{:});
     newitem=rmfield(newitem,N('_ArrayData_'));
     if(varargin{1}.base64)
         newitem.(N('_ArrayZipData_'))=char(base64encode(newitem.(N('_ArrayZipData_'))));

jload.m

@@ -60,15 +60,6 @@
 
 ws=jsonopt('ws','caller',opt);
 
-loadfun=@loadbj;
-if(regexp(filename,'\.[jJ][sS][oO][nN]$'))
-    loadfun=@loadjson;
-elseif(regexp(filename,'\.[jJ][dD][tT]$'))
-    loadfun=@loadjson;
-elseif(regexp(filename,'\.[mM][sS][gG][pP][kK]$'))
-    loadfun=@loadmsgpack;
-end
-
 if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
     jsonstr=fileread(filename);
     pos=regexp(jsonstr,'}\n\n\n{"WorkspaceData":','once');
@@ -78,9 +69,9 @@
     header=jsondecode(jsonstr(1:pos+1));
 else
     try
-        header=loadfun(filename,'ObjectID',1, 'MaxBuffer', 65536, varargin{:});
+        header=loadjd(filename,'ObjectID',1, 'MaxBuffer', 65536, varargin{:});
     catch
-        header=loadfun(filename,'ObjectID',1, varargin{:});
+        header=loadjd(filename,'ObjectID',1, varargin{:});
     end
 end
 
@@ -97,7 +88,7 @@
 if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
     body=jdatadecode(jsondecode(jsonstr(pos+4:end)));
 else
-    body=loadfun(filename,'ObjectID',2, varargin{:});
+    body=loadjd(filename,'ObjectID',2, varargin{:});
 end
 
 if(nargout==0)

jsave.m

@@ -49,6 +49,22 @@
 end
 
 opt=varargin2struct(varargin{:});
+if(~isfield(opt,'nthread'))
+    opt.nthread=4;
+end
+if(~isfield(opt,'compression'))
+    if(exist('zipmat'))
+        opt.compression='blosc2zstd';
+    else
+        opt.compression='zlib';
+    end
+end
+if(~isfield(opt,'shuffle'))
+    opt.shuffle=1;
+end
+if(~isfield(opt,'typesize'))
+    opt.typesize=4;
+end
 
 ws=jsonopt('ws','caller',opt);
 
@@ -90,22 +106,13 @@
     body.(varlist{i})=evalin(ws,varlist{i});
 end
 
-savefun=@savebj;
-loadfun=@loadbj;
-if(regexp(filename,'\.[jJ][sS][oO][nN]$'))
-    savefun=@savejson;
-    loadfun=@loadjson;
-elseif(regexp(filename,'\.[jJ][dD][tT]$'))
-    savefun=@savejson;
-    loadfun=@loadjson;
-elseif(regexp(filename,'\.[mM][sS][gG][pP][kK]$'))
-    savefun=@savemsgpack;
-end
-
 if(nargout==1)
     varargout{1}=header;
 end
 
+defaultopt={'compression',opt.compression,'nthread',opt.nthread,...
+    'shuffle',opt.shuffle,'typesize',opt.typesize,'keeptype',1,'array2struct',1};
+
 if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
     if(isempty(regexp(filename,'\.[jJ][sS][oO][nN]$', 'once')))
         filename=regexprep(filename,'\.[a-zA-Z]+$','.jdt');
@@ -115,8 +122,7 @@
     clear output;
 
     output.WorkspaceData=jdataencode(body,'AnnotateArray',1,'base64',1,...
-       'Compression','zlib','UseArrayZipSize',1,'MapAsStruct',1,...
-       'prefix','x',varargin{:});
+       'UseArrayZipSize',1,'MapAsStruct',1,'prefix','x',defaultopt{:},varargin{:});
     bodyjson=jsonencode(output);
     clear output;
 
@@ -126,15 +132,15 @@
     fwrite(fid,bodyjson);
     fclose(fid);
 elseif(jsonopt('usemmap',0,opt)==1)
-    bodyjson=savefun('WorkspaceData',body,...
-        'compression','zlib','keeptype',1,'array2struct',1,varargin{:});
-    header.(encodevarname('_MMap_'))=loadfun(bodyjson,'mmaponly',1,varargin{:});
-    savefun('WorkspaceHeader',header,'filename',filename,varargin{:});
+    bodyjson=savejd('WorkspaceData',body,...
+        defaultopt{:},varargin{:});
+    header.(encodevarname('_MMap_'))=loadjd(bodyjson,'mmaponly',1,varargin{:});
+    savejd('WorkspaceHeader',header,'filename',filename,varargin{:});
     fid=fopen(filename,'ab+');
     fwrite(fid,bodyjson);
     fclose(fid);
 else
-    savefun('WorkspaceHeader',header,'filename',filename,varargin{:});
-    savefun('WorkspaceData',body,'filename',filename,'append',1,...
-        'compression','zlib','keeptype',1,'array2struct',1,varargin{:});
+    savejd('WorkspaceHeader',header,'filename',filename,varargin{:});
+    savejd('WorkspaceData',body,'filename',filename,'append',1,...
+        defaultopt{:},varargin{:});
 end

loadjd.m

@@ -13,7 +13,7 @@
 %    input:
 %        inputfile: the input hierarchical container data file, supporting:
 %                *.json,.jnii,.jdt,.jmsh,.jnirs,.jbids: JSON/JData based data files, see https://neurojson.org/jdata/draft2
-%                *.bjd,.bnii,.jdb,.bmsh,.bnirs: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
+%                *.bjd,.bnii,.jdb,.bmsh,.bnirs,.pmat: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
 %                *.ubj: UBJSON-encoded files, see http://ubjson.org
 %                *.msgpack: MessagePack-encoded files, see http://msgpack.org
 %                *.h5,.hdf5,.snirf: HDF5 files, see https://www.hdfgroup.org/
@@ -45,7 +45,7 @@
 
 if(regexpi(filename,'\.json$|\.jnii$|\.jdt$|\.jdat$|\.jmsh$|\.jnirs|\.jbids$'))
     [varargout{1:nargout}]=loadjson(filename,varargin{:});
-elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$'))
+elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$|\.pmat'))
     [varargout{1:nargout}]=loadbj(filename,varargin{:});
 elseif(regexpi(filename,'\.ubj$'))
     [varargout{1:nargout}]=loadubjson(filename,varargin{:});
@@ -57,5 +57,5 @@
     end
     [varargout{1:nargout}]=loadh5(filename,varargin{:});
 else
-    error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf');
+    error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf,.pmat');
 end

savebj.m

@@ -656,8 +656,14 @@
         cid=I_(uint32(max(size(fulldata))),varargin{:});
         txt=[txt, N_('_ArrayZipSize_',opt),I_a(size(fulldata),cid(1),varargin{:})];
         txt=[txt, N_('_ArrayZipType_',opt),S_(dozip,opt)];
-	compfun=str2func([dozip 'encode']);
-	txt=[txt,N_('_ArrayZipData_',opt), I_a(compfun(typecast(fulldata(:),'uint8')),Imarker(1),varargin{:})];
+        encodeparam={};
+        if(~isempty(regexp(dozip,'^blosc2', 'once')))
+            compfun=@blosc2encode;
+            encodeparam={dozip, 'nthread', jsonopt('nthread',1,opt), 'shuffle', jsonopt('shuffle',1,opt), 'typesize', jsonopt('typesize',length(typecast(fulldata(1),'uint8')),opt)};
+        else
+            compfun=str2func([dozip 'encode']);
+        end
+        txt=[txt,N_('_ArrayZipData_',opt), I_a(compfun(typecast(fulldata(:),'uint8'),encodeparam{:}),Imarker(1),varargin{:})];
         childcount=childcount+3;
     else
         if(ismsgpack)

savejd.m

@@ -21,7 +21,7 @@
 %             class instance).
 %        outputfile: the output file name to the hierarchical container file
 %                *.json,.jnii,.jdt,.jmsh,.jnirs,.jbids: JSON/JData based data files, see https://neurojson.org/jdata/draft2
-%                *.bjd,.bnii,.jdb,.bmsh,.bnirs: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
+%                *.bjd,.bnii,.jdb,.bmsh,.bnirs,.pmat: binary JData (BJData) files, see https://neurojson.org/bjdata/draft2
 %                *.ubj: UBJSON-encoded files, see http://ubjson.org
 %                *.msgpack: MessagePack-encoded files, see http://msgpack.org
 %                *.h5,.hdf5,.snirf: HDF5 files, see https://www.hdfgroup.org/
@@ -66,7 +66,7 @@
 end
 if(regexpi(filename,'\.json$|\.jnii$|\.jdt$|\.jdat$|\.jmsh$|\.jnirs$|\.jbids$'))
     [varargout{1:nargout}]=savejson(varargin{:});
-elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$'))
+elseif(regexpi(filename,'\.bjd$|\.bnii$|\.jdb$|\.jbat$|\.bmsh$|\.bnirs$|\.pmat'))
     [varargout{1:nargout}]=savebj(varargin{:});
 elseif(regexpi(filename,'\.ubj$'))
     [varargout{1:nargout}]=saveubjson(varargin{:});
@@ -85,5 +85,5 @@
     end
     [varargout{1:nargout}]=saveh5(varargin{2},filename, opt);
 else
-    error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf');
+    error('file suffix must be one of .json,.jnii,.jdt,.jmsh,.jnirs,.jbids,.bjd,.bnii,.jdb,.bmsh,.bnirs,.ubj,.msgpack,.h5,.hdf5,.snirf,.pmat');
 end

savejson.m

@@ -565,35 +565,36 @@
 if (~isnumeric(item) && ~islogical(item) && ~ischar(item))
     error('input is not an array');
 end
-ws = varargin{1}.whitespaces_;
+opt=varargin{1};
+ws = opt.whitespaces_;
 padding1 = repmat(ws.tab, 1, level);
 padding0 = repmat(ws.tab, 1, level + 1);
 nl = ws.newline;
 sep = ws.sep;
 
-dozip = varargin{1}.compression;
-zipsize = varargin{1}.compressarraysize;
-format = varargin{1}.formatversion;
-isnest = varargin{1}.nestarray;
+dozip = opt.compression;
+zipsize = opt.compressarraysize;
+format = opt.formatversion;
+isnest = opt.nestarray;
 
-if (~varargin{1}.nosubstruct_ && (((isnest == 0) && length(size(item)) > 2) || issparse(item) || ~isreal(item) || ...
-                                  (isempty(item) && any(size(item))) || varargin{1}.arraytostruct || (~isempty(dozip) && numel(item) > zipsize)))
+if (~opt.nosubstruct_ && (((isnest == 0) && length(size(item)) > 2) || issparse(item) || ~isreal(item) || ...
+                                  (isempty(item) && any(size(item))) || opt.arraytostruct || (~isempty(dozip) && numel(item) > zipsize)))
     if (isempty(name))
         txt = sprintf('%s{%s%s"_ArrayType_":"%s",%s%s"_ArraySize_":%s,%s', ...
                       padding1, nl, padding0, class(item), nl, padding0, regexprep(mat2str(size(item)), '\s+', ','), nl);
     else
         txt = sprintf('%s"%s":{%s%s"_ArrayType_":"%s",%s%s"_ArraySize_":%s,%s', ...
-                      padding1, decodevarname(name, varargin{1}.unpackhex), nl, padding0, class(item), nl, padding0, regexprep(mat2str(size(item)), '\s+', ','), nl);
+                      padding1, decodevarname(name, opt.unpackhex), nl, padding0, class(item), nl, padding0, regexprep(mat2str(size(item)), '\s+', ','), nl);
     end
 else
     numtxt = matdata2json(item, level + 1, varargin{:});
     if (isempty(name))
         txt = sprintf('%s%s', padding1, numtxt);
     else
         if (numel(item) == 1 && varargin{1}.singletarray == 0)
-            txt = sprintf('%s"%s":%s', padding1, decodevarname(name, varargin{1}.unpackhex), numtxt);
+            txt = sprintf('%s"%s":%s', padding1, decodevarname(name, opt.unpackhex), numtxt);
         else
-            txt = sprintf('%s"%s":%s', padding1, decodevarname(name, varargin{1}.unpackhex), numtxt);
+            txt = sprintf('%s"%s":%s', padding1, decodevarname(name, opt.unpackhex), numtxt);
         end
     end
     return
@@ -659,8 +660,14 @@
         end
         txt = sprintf(dataformat, txt, padding0, '"_ArrayZipSize_":', regexprep(mat2str(size(fulldata)), '\s+', ','), sep);
         txt = sprintf(dataformat, txt, padding0, '"_ArrayZipType_":"', dozip, ['"' sep]);
-        compfun = str2func([dozip 'encode']);
-        txt = sprintf(dataformat, txt, padding0, '"_ArrayZipData_":"', char(base64encode(compfun(typecast(fulldata(:), 'uint8')))), ['"' nl]);
+        encodeparam={};
+        if(~isempty(regexp(dozip,'^blosc2', 'once')))
+            compfun=@blosc2encode;
+            encodeparam={dozip, 'nthread', jsonopt('nthread',1,opt), 'shuffle', jsonopt('shuffle',1,opt), 'typesize', jsonopt('typesize',length(typecast(fulldata(1),'uint8')),opt)};
+        else
+            compfun=str2func([dozip 'encode']);
+        end
+        txt = sprintf(dataformat, txt, padding0, '"_ArrayZipData_":"', char(base64encode(compfun(typecast(fulldata(:), 'uint8'),encodeparam{:}))), ['"' nl]);
     else
         if (isreal(item))
             txt = sprintf(dataformat, txt, padding0, '"_ArrayData_":', ...