version bump 0.9.10: nodejs performance

- xlsx.njs use dense mode by default - CSV Streaming output for nodejs - XLSX/XLSB error on password protected documents - record hoppers switch on record number rather than name
wockesha3120 · Apr 9, 2017 · 51182e5 · 51182e5
1 parent f43caca
commit 51182e5
Show file tree

Hide file tree

Showing 37 changed files with 2,060 additions and 1,199 deletions.
diff --git a/.gitignore b/.gitignore
@@ -5,7 +5,7 @@ misc/coverage.html
 misc/prof.js
 v8.log
 tmp
-*.txt
+*.[tT][xX][tT]
 *.[cC][sS][vV]
 *.[dD][iIbB][fF]
 *.[pP][rR][nN]

diff --git a/.npmignore b/.npmignore
@@ -8,7 +8,7 @@ node_modules
 _book
 book.json
 tmp
-*.txt
+*.[tT][xX][tT]
 *.[cC][sS][vV]
 *.[dD][iIbB][fF]
 *.[pP][rR][nN]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,9 +5,9 @@ but not limited to API changes and file location changes.  Minor behavioral
 changes may not be included if they are not expected to break existing code.
 
 
-## Unreleased
+## 0.9.10 (2017-04-08)
 
-*
+* `--perf` renamed to `--read-only`
 
 ## 0.9.9 (2017-04-03)
 

diff --git a/bin/xlsx.njs b/bin/xlsx.njs
@@ -37,7 +37,7 @@ program
 	.option('-n, --sheet-rows <num>', 'Number of rows to process (0=all rows)')
 	.option('--sst', 'generate shared string table for XLS* formats')
 	.option('--compress', 'use compression when writing XLSX/M/B and ODS')
-	.option('--perf', 'do not generate output')
+	.option('--read-only', 'do not generate output')
 	.option('--all', 'parse everything; write as much as possible')
 	.option('--dev', 'development mode')
 	.option('--sparse', 'sparse mode')
@@ -58,17 +58,6 @@ var wb_formats_2 = [
 ];
 program.parse(process.argv);
 
-/* see https://github.com/SheetJS/j/issues/4 */
-if(process.version === 'v0.10.31') {
-	var msgs = [
-		"node v0.10.31 is known to crash on OSX and Linux, refusing to proceed.",
-		"see https://github.com/SheetJS/j/issues/4 for the relevant discussion.",
-		"see https://github.com/joyent/node/issues/8208 for the relevant node issue"
-	];
-	msgs.forEach(function(m) { console.error(m); });
-	process.exit(1);
-}
-
 var filename/*:?string*/, sheetname = '';
 if(program.args[0]) {
 	filename = program.args[0];
@@ -165,7 +154,7 @@ try {
 	process.exit(4);
 }
 
-if(program.perf) process.exit(0);
+if(program.readOnly) process.exit(0);
 
 /* single worksheet formats */
 [
@@ -181,14 +170,22 @@ if(program.perf) process.exit(0);
 } });
 
 var oo = "";
+var strm = false;
 if(!program.quiet) console.error(target_sheet);
 if(program.formulae) oo = X.utils.get_formulae(ws).join("\n");
 else if(program.json) oo = JSON.stringify(X.utils.sheet_to_row_object_array(ws));
 else if(program.rawJs) oo = JSON.stringify(X.utils.sheet_to_row_object_array(ws,{raw:true}));
 else if(program.arrays) oo = JSON.stringify(X.utils.sheet_to_row_object_array(ws,{raw:true, header:1}));
-else oo = X.utils.make_csv(ws, {FS:program.fieldSep, RS:program.rowSep});
+else {
+	strm = true;
+	var stream = X.stream.to_csv(ws, {FS:program.fieldSep, RS:program.rowSep});
+	if(program.output) stream.pipe(fs.createWriteStream(program.output));
+	else stream.pipe(process.stdout);
+}
 
-if(program.output) fs.writeFileSync(program.output, oo);
-else console.log(oo);
+if(!strm) {
+	if(program.output) fs.writeFileSync(program.output, oo);
+	else console.log(oo);
+}
 /*::   } */
 /*:: } */
diff --git a/bits/01_version.js b/bits/01_version.js
@@ -1 +1 @@
-XLSX.version = '0.9.9';
+XLSX.version = '0.9.10';
diff --git a/bits/03_dense.js b/bits/03_dense.js
@@ -1 +1 @@
-var DENSE = false;
+var DENSE = null;
diff --git a/bits/09_types.js b/bits/09_types.js
@@ -7,7 +7,7 @@ declare type BufArray = {
 	push(buf:Block):void;
 };
 
-type RecordHopperCB = {(d:any, R:any, RT:number):?boolean;};
+type RecordHopperCB = {(d:any, Rn:string, RT:number):?boolean;};
 
 type EvertType = {[string]:string};
 type EvertNumType = {[string]:number};

diff --git a/bits/24_hoppers.js b/bits/24_hoppers.js
@@ -3,17 +3,18 @@ function recordhopper(data, cb/*:RecordHopperCB*/, opts/*:?any*/) {
 	if(!data) return;
 	var tmpbyte, cntbyte, length;
 	prep_blob(data, data.l || 0);
-	while(data.l < data.length) {
-		var RT = data.read_shift(1);
+	var L = data.length, RT = 0, tgt = 0;
+	while(data.l < L) {
+		RT = data.read_shift(1);
 		if(RT & 0x80) RT = (RT & 0x7F) + ((data.read_shift(1) & 0x7F)<<7);
 		var R = XLSBRecordEnum[RT] || XLSBRecordEnum[0xFFFF];
 		tmpbyte = data.read_shift(1);
 		length = tmpbyte & 0x7F;
 		for(cntbyte = 1; cntbyte <4 && (tmpbyte & 0x80); ++cntbyte) length += ((tmpbyte = data.read_shift(1)) & 0x7F)<<(7*cntbyte);
-		var tgt = data.l + length;
+		tgt = data.l + length;
 		var d = R.f(data, length, opts);
 		data.l = tgt;
-		if(cb(d, R, RT)) return;
+		if(cb(d, R.n, RT)) return;
 	}
 }
 

diff --git a/bits/27_csfutils.js b/bits/27_csfutils.js
@@ -6,7 +6,7 @@ function sheet_to_workbook(sheet/*:Worksheet*/, opts)/*:Workbook*/ {
 
 function aoa_to_sheet(data/*:AOA*/, opts/*:?any*/)/*:Worksheet*/ {
 	var o = opts || {};
-	if(DENSE != null) o.dense = DENSE;
+	if(DENSE != null && o.dense == null) o.dense = DENSE;
 	var ws/*:Worksheet*/ = o.dense ? ([]/*:any*/) : ({}/*:any*/);
 	var range/*:Range*/ = ({s: {c:10000000, r:10000000}, e: {c:0, r:0}}/*:any*/);
 	for(var R = 0; R != data.length; ++R) {

diff --git a/bits/40_harb.js b/bits/40_harb.js
@@ -408,7 +408,7 @@ var PRN = (function() {
 	function dsv_to_sheet_str(str/*:string*/, opts)/*:Worksheet*/ {
 		var o = opts || {};
 		var sep = "";
-		if(DENSE != null) o.dense = DENSE;
+		if(DENSE != null && o.dense == null) o.dense = DENSE;
 		var ws/*:Worksheet*/ = o.dense ? ([]/*:any*/) : ({}/*:any*/);
 		var range/*:Range*/ = ({s: {c:0, r:0}, e: {c:0, r:0}}/*:any*/);
 

diff --git a/bits/41_lotus.js b/bits/41_lotus.js
@@ -10,7 +10,7 @@ var WK_ = (function() {
 			var tgt = data.l + length;
 			var d = R.f(data, length, opts);
 			data.l = tgt;
-			if(cb(d, R, RT)) return;
+			if(cb(d, R.n, RT)) return;
 		}
 	}
 
@@ -27,7 +27,7 @@ var WK_ = (function() {
 	function lotus_to_workbook_buf(d,opts)/*:Workbook*/ {
 		if(!d) return d;
 		var o = opts || {};
-		if(DENSE != null) o.dense = DENSE;
+		if(DENSE != null && o.dense == null) o.dense = DENSE;
 		var s = (o.dense ? [] : {}), n = "Sheet1", sidx = 0;
 		var sheets = {}, snames = [n];
 
@@ -37,7 +37,7 @@ var WK_ = (function() {
 		else if(d[2] == 0x1a) o.Enum = WK3Enum;
 		else if(d[2] == 0x0e) { o.Enum = WK3Enum; o.qpro = true; d.l = 0; }
 		else throw new Error("Unrecognized LOTUS BOF " + d[2]);
-		lotushopper(d, function(val, R, RT) {
+		lotushopper(d, function(val, Rn, RT) {
 			if(d[2] == 0x02) switch(RT) {
 				case 0x00:
 					o.vers = val;

diff --git a/bits/43_sstbin.js b/bits/43_sstbin.js
@@ -7,14 +7,24 @@ function parse_BrtBeginSst(data, length) {
 function parse_sst_bin(data, opts)/*:SST*/ {
 	var s/*:SST*/ = ([]/*:any*/);
 	var pass = false;
-	recordhopper(data, function hopper_sst(val, R, RT) {
-		switch(R.n) {
-			case 'BrtBeginSst': s.Count = val[0]; s.Unique = val[1]; break;
-			case 'BrtSSTItem': s.push(val); break;
-			case 'BrtEndSst': return true;
-			case 'BrtFRTBegin': pass = true; break;
-			case 'BrtFRTEnd': pass = false; break;
-			default: if(!pass || opts.WTF) throw new Error("Unexpected record " + RT + " " + R.n);
+	recordhopper(data, function hopper_sst(val, R_n, RT) {
+		switch(RT) {
+			case 0x009F: /* 'BrtBeginSst' */
+				s.Count = val[0]; s.Unique = val[1]; break;
+			case 0x0013: /* 'BrtSSTItem' */
+				s.push(val); break;
+			case 0x00A0: /* 'BrtEndSst' */
+				return true;
+
+			case 0x0023: /* 'BrtFRTBegin' */
+				pass = true; break;
+			case 0x0024: /* 'BrtFRTEnd' */
+				pass = false; break;
+
+			default:
+				if(R_n.indexOf("Begin") > 0) state.push(R_n);
+				else if(R_n.indexOf("End") > 0) state.pop();
+				if(!pass || opts.WTF) throw new Error("Unexpected record " + RT + " " + R_n);
 		}
 	});
 	return s;

diff --git a/bits/48_stybin.js b/bits/48_stybin.js
@@ -49,41 +49,47 @@ function parse_sty_bin(data, themes, opts) {
 	styles.CellXf = [];
 	var state = [];
 	var pass = false;
-	recordhopper(data, function hopper_sty(val, R, RT) {
-		switch(R.n) {
-			case 'BrtFmt':
+	recordhopper(data, function hopper_sty(val, R_n, RT) {
+		switch(RT) {
+			case 0x002C: /* 'BrtFmt' */
 				styles.NumberFmt[val[0]] = val[1]; SSF.load(val[1], val[0]);
 				break;
-			case 'BrtFont': break; /* TODO */
-			case 'BrtKnownFonts': break; /* TODO */
-			case 'BrtFill': break; /* TODO */
-			case 'BrtBorder': break; /* TODO */
-			case 'BrtXF':
+			case 0x002B: /* 'BrtFont' */ break;
+			case 0x0401: /* 'BrtKnownFonts' */ break;
+			case 0x002D: /* 'BrtFill' */ break;
+			case 0x002E: /* 'BrtBorder' */ break;
+			case 0x002F: /* 'BrtXF' */
 				if(state[state.length - 1] == "BrtBeginCellXFs") {
 					styles.CellXf.push(val);
 				}
-				break; /* TODO */
-			case 'BrtStyle': break; /* TODO */
-			case 'BrtDXF': break; /* TODO */
-			case 'BrtMRUColor': break; /* TODO */
-			case 'BrtIndexedColor': break; /* TODO */
+				break;
+			case 0x0030: /* 'BrtStyle' */
+			case 0x01FB: /* 'BrtDXF' */
+			case 0x023C: /* 'BrtMRUColor' */
+			case 0x01DB: /* 'BrtIndexedColor': */
+				break;
 
-			case 'BrtDXF14': break;
-			case 'BrtDXF15': break;
-			case 'BrtUid': break;
-			case 'BrtSlicerStyleElement': break;
-			case 'BrtTableStyleElement': break;
-			case 'BrtTimelineStyleElement': break;
+			case 0x0493: /* 'BrtDXF14' */
+			case 0x0836: /* 'BrtDXF15' */
+			case 0x046A: /* 'BrtSlicerStyleElement' */
+			case 0x0200: /* 'BrtTableStyleElement' */
+			case 0x082F: /* 'BrtTimelineStyleElement' */
+			/* case 'BrtUid' */
+				break;
 
-			case 'BrtFRTBegin': pass = true; break;
-			case 'BrtFRTEnd': pass = false; break;
-			case 'BrtACBegin': state.push(R.n); break;
-			case 'BrtACEnd': state.pop(); break;
+			case 0x0023: /* 'BrtFRTBegin' */
+				pass = true; break;
+			case 0x0024: /* 'BrtFRTEnd' */
+				pass = false; break;
+			case 0x0025: /* 'BrtACBegin' */
+				state.push(R_n); break;
+			case 0x0026: /* 'BrtACEnd' */
+				state.pop(); break;
 
 			default:
-				if((R.n||"").indexOf("Begin") > 0) state.push(R.n);
-				else if((R.n||"").indexOf("End") > 0) state.pop();
-				else if(!pass || opts.WTF) throw new Error("Unexpected record " + RT + " " + R.n);
+				if((R_n||"").indexOf("Begin") > 0) state.push(R_n);
+				else if((R_n||"").indexOf("End") > 0) state.pop();
+				else if(!pass || opts.WTF) throw new Error("Unexpected record " + RT + " " + R_n);
 		}
 	});
 	return styles;

diff --git a/bits/53_ccbin.js b/bits/53_ccbin.js
@@ -16,12 +16,15 @@ function parse_BrtCalcChainItem$(data, length) {
 function parse_cc_bin(data, opts) {
 	var out = [];
 	var pass = false;
-	recordhopper(data, function hopper_cc(val, R, RT) {
-		switch(R.n) {
-			case 'BrtCalcChainItem$': out.push(val); break;
-			case 'BrtBeginCalcChain$': break;
-			case 'BrtEndCalcChain$': break;
-			default: if(!pass || opts.WTF) throw new Error("Unexpected record " + RT + " " + R.n);
+	recordhopper(data, function hopper_cc(val, R_n, RT) {
+		switch(RT) {
+			case 0x003F: /* 'BrtCalcChainItem$' */
+				out.push(val); break;
+
+			default:
+				if((R_n||"").indexOf("Begin") > 0){}
+				else if((R_n||"").indexOf("End") > 0){}
+				else if(!pass || opts.WTF) throw new Error("Unexpected record " + RT + " " + R_n);
 		}
 	});
 	return out;

diff --git a/bits/58_cmntbin.js b/bits/58_cmntbin.js
@@ -31,28 +31,35 @@ function parse_comments_bin(data, opts) {
 	var authors = [];
 	var c = {};
 	var pass = false;
-	recordhopper(data, function hopper_cmnt(val, R, RT) {
-		switch(R.n) {
-			case 'BrtCommentAuthor': authors.push(val); break;
-			case 'BrtBeginComment': c = val; break;
-			case 'BrtCommentText': c.t = val.t; c.h = val.h; c.r = val.r; break;
-			case 'BrtEndComment':
+	recordhopper(data, function hopper_cmnt(val, R_n, RT) {
+		switch(RT) {
+			case 0x0278: /* 'BrtCommentAuthor' */
+				authors.push(val); break;
+			case 0x027B: /* 'BrtBeginComment' */
+				c = val; break;
+			case 0x027D: /* 'BrtCommentText' */
+				c.t = val.t; c.h = val.h; c.r = val.r; break;
+			case 0x027C: /* 'BrtEndComment' */
 				c.author = authors[c.iauthor];
 				delete c.iauthor;
 				if(opts.sheetRows && opts.sheetRows <= c.rfx.r) break;
 				if(!c.t) c.t = "";
 				delete c.rfx; out.push(c); break;
 
-			case 'BrtUid': break;
-			case 'BrtFRTBegin': pass = true; break;
-			case 'BrtFRTEnd': pass = false; break;
-			case 'BrtACBegin': break;
-			case 'BrtACEnd': break;
+			/* case 'BrtUid': */
+
+			case 0x0023: /* 'BrtFRTBegin' */
+				pass = true; break;
+			case 0x0024: /* 'BrtFRTEnd' */
+				pass = false; break;
+			case 0x0025: /* 'BrtACBegin' */ break;
+			case 0x0026: /* 'BrtACEnd' */ break;
+
 
 			default:
-				if((R.n||"").indexOf("Begin") > 0){}
-				else if((R.n||"").indexOf("End") > 0){}
-				else if(!pass || opts.WTF) throw new Error("Unexpected record " + RT + " " + R.n);
+				if((R_n||"").indexOf("Begin") > 0){}
+				else if((R_n||"").indexOf("End") > 0){}
+				else if(!pass || opts.WTF) throw new Error("Unexpected record " + RT + " " + R_n);
 		}
 	});
 	return out;

diff --git a/bits/67_wsxml.js b/bits/67_wsxml.js
@@ -4,13 +4,13 @@ function parse_ws_xml_dim(ws, s) {
 }
 var mergecregex = /<(?:\w:)?mergeCell ref="[A-Z0-9:]+"\s*[\/]?>/g;
 var sheetdataregex = /<(?:\w+:)?sheetData>([^\u2603]*)<\/(?:\w+:)?sheetData>/;
-var hlinkregex = /<(?:\w*:)?hyperlink [^>]*>/mg;
+var hlinkregex = /<(?:\w:)?hyperlink [^>]*>/mg;
 var dimregex = /"(\w*:\w*)"/;
-var colregex = /<(?:\w*:)?col[^>]*[\/]?>/g;
+var colregex = /<(?:\w:)?col[^>]*[\/]?>/g;
 /* 18.3 Worksheets */
 function parse_ws_xml(data/*:?string*/, opts, rels, wb, themes, styles)/*:Worksheet*/ {
 	if(!data) return data;
-	if(DENSE != null) opts.dense = DENSE;
+	if(DENSE != null && opts.dense == null) opts.dense = DENSE;
 	/* 18.3.1.99 worksheet CT_Worksheet */
 	var s = opts.dense ? ([]/*:any*/) : ({}/*:any*/);