Skip to content

Commit

Permalink
hack fixing decodeString problem
Browse files Browse the repository at this point in the history
  • Loading branch information
@thlorenz
thlorenz committed Dec 8, 2013
1 parent 534f012 commit 83ec090
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 51 deletions.
69 changes: 52 additions & 17 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,32 @@ function outForFiles(getOutStream, outdir) {
var outfile = outdir ? path.join(outdir, entry.path) : null
, outStream = getOutStream(outfile, outdir, entry.relative);

this.push({ file: entry.fullPath, outStream: outStream });
this.push({ file: entry.fullPath, outfile: outfile, outStream: outStream });
cb();
}
}

function ensureNoStringDecode(s) {
if (s._writableState) s._writableState.decodeStrings = false;

// Oh what an ugly hack :( , but somehow just setting decodeStrings fails in cases
// In particular test/transforms.js: "running trimLeading and then toUpper transforms" fails without this
var write_ = s._write.bind(s);
s._write = function (chunk, enc, cb) {
if (enc === 'utf8') write_(chunk, enc, cb);
else write_(chunk.toString(), 'utf8', cb);
}
}

function transformStream(transforms) {
return transforms && transforms.length
? function (file) {
var streams = transforms.map(function (t) { return t(file) });
var streams = transforms.map(function (t) {
var s = t(file)
ensureNoStringDecode(s);
return s;
});
var combined = combine.apply(null, streams);
combined._writableState.decodeStrings = false;
return combined;
}
: function (file) { return through() };
Expand All @@ -48,11 +63,16 @@ function transformStream(transforms) {
function transformContent(transforms) {
var ts = transformStream(transforms);
return function (entry, enc, cb) {
var self = this;

fs.createReadStream(entry.file, { encoding: 'utf8' })
.on('error', cb)
.pipe(ts(entry.file))
.on('error', cb)
.on('end', cb)
.on('end', function () {
self.push({ file: entry.file, outfile: entry.outfile });
cb();
})
.pipe(entry.outStream);
}
}
Expand All @@ -79,16 +99,16 @@ function keepName(outfile, outdir, relative) { return outfile }
var go = module.exports = function(mutinyopts, readopts) {
var transforms
, outdir = mutinyopts.outdir
, stream = new PassThrough({ objectMode: true });
, progress = new PassThrough({ objectMode: true });

readopts = readopts || {}

if (!outdir && !mutinyopts.getOutStream) {
stream.emit('error', new Error('Need to supply the outdir option (full path to where to store transformed files) or provide custom outStream function.'));
progress.emit('error', new Error('Need to supply the outdir option (full path to where to store transformed files) or provide custom outStream function.'));
}

if (mutinyopts.getOutStream && mutinyopts.rename) {
stream.emit('error', new Error('If you already supply the outstream it doesn\'t make any sense to also supply a rename function'));
progress.emit('error', new Error('If you already supply the outstream it doesn\'t make any sense to also supply a rename function'));
}

var rename = mutinyopts.rename || keepName;
Expand All @@ -99,15 +119,16 @@ var go = module.exports = function(mutinyopts, readopts) {
}

readdirp(readopts)
.on('warn', stream.emit.bind(stream, 'warn'))
.on('error', stream.emit.bind(stream, 'error'))
.on('warn', progress.emit.bind(progress, 'warn'))
.on('error', progress.emit.bind(progress, 'error'))
.pipe(through({ objectMode: true }, outForFiles(getOutStream, outdir)))
.on('error', stream.emit.bind(stream, 'error'))
.on('error', progress.emit.bind(progress, 'error'))
.pipe(through({ objectMode: true }, transformContent(transforms)))
.on('error', stream.emit.bind(stream, 'error'))
.pipe(stream);
.on('error', progress.emit.bind(progress, 'error'))
.on('end', progress.emit.bind(progress, 'end'))
.pipe(progress);

return stream;
return progress;
};

function inspect(obj, depth) {
Expand All @@ -117,15 +138,29 @@ function inspect(obj, depth) {
function toUpper(file, content) {
return through(
function (chunk, enc, cb) {
console.log('encoding', enc);
this.push(chunk.toUpperCase());
cb();
}
)
}

function trimLeading(file, content, cb) {
var c = content.replace(/^\s+/mg, '');
cb(null, c);
var data = '';

function ondata(chunk, enc, cb) {
console.log('trim-encoding', enc);
data += chunk;
cb();
}

function onend(cb) {
var c = data.replace(/^\s+/mg, '');
this.push(c);
cb();
}

return through(ondata, onend);
}

function getStdOut () { return process.stdout }
Expand All @@ -138,9 +173,9 @@ if (!module.parent && typeof window === 'undefined') {
var outdir = path.join(fixtures, 'out');


go({ transforms: [ toUpper ], outdir: outdir }, { root: root })
go({ transforms: [ trimLeading, toUpper ], getOutStream: getStdOut, outdir: outdir }, { root: root })
.on('data', function (data) {
console.log('data', data);
console.log(data);
})
.on('error', function (err) {
console.error('error', err);
Expand Down
107 changes: 77 additions & 30 deletions test/transform.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,44 @@

var test = require('tap').test
var path = require('path');
var through = require('through2');
var mutiny = require('../')
var adapt = require('./util/adapt-entries');

function toUpper(file, content, cb) {
cb(null, content.toUpperCase());
function toUpper(file, content) {
return through(
function (chunk, enc, cb) {
this.push(chunk.toUpperCase());
cb();
}
)
}

function toUpperError(file, content, cb) {
if ((/two/mg).test(content)) return cb(new Error('I hate to be number two!'));
cb(null, content.toUpperCase());
return through(
function (chunk, enc, cb) {
if ((/two/mg).test(chunk)) return cb(new Error('I hate to be number two!'));
this.push(chunk.toUpperCase());
cb();
}
)
}

function trimLeading(file, content, cb) {
var c = content.replace(/^\s+/mg, '');
cb(null, c);
var data = '';

function ondata(chunk, enc, cb) {
data += chunk;
cb();
}

function onend(cb) {
var c = data.replace(/^\s+/mg, '');
this.push(c);
cb();
}

return through(ondata, onend);
}

var root = path.join(__dirname, '..', 'test', 'fixtures', 'root');
Expand All @@ -31,58 +54,82 @@ function inspect(obj, depth) {
console.error(require('util').inspect(obj, false, depth || 5, true));
}

function getStdOut () { return process.stdout }

test('\nrunning upperCase transform', function (t) {
var data = []
var progress = []

mutiny({ root: root }, toUpper)
mutiny({ getOutStream: getStdOut, transforms: toUpper }, { root: root })
.on('error', fail.bind(t))
.on('data', [].push.bind(data))
.on('data', [].push.bind(progress))
.on('end', function () {
t.deepEqual(
adapt(data)
adapt(progress)
, [ { file: '/fixtures/root/index.html',
content: '<HTML>\n <BODY>\n <H1>INDEX</H1> \n </BODY>\n</HTML>\n' },
{ file: '/fixtures/root/sub1/one.html',
content: '<HTML>\n <BODY>\n <H1>ONE</H1> \n </BODY>\n</HTML>\n' },
{ file: '/fixtures/root/sub2/two.html',
content: '<HTML>\n <BODY>\n <H1>TWO</H1> \n </BODY>\n</HTML>\n' } ]
, 'uppercases all content'
outfile: null },
{ file: '/fixtures/root/sub1/one.html',
outfile: null },
{ file: '/fixtures/root/sub2/two.html',
outfile: null } ]
, 'reports progress for all files'
)

t.end()
});
})

test('\nrunning upperCase and trimLeading transforms', function (t) {
var data = []
test('\nrunning toUpper and then trimLeading transforms', function (t) {
var progress = []

mutiny({ root: root }, [ toUpper, trimLeading ])
mutiny({ getOutStream: getStdOut, transforms: [ toUpper, trimLeading ] }, { root: root })
.on('error', fail.bind(t))
.on('data', [].push.bind(data))
.on('data', [].push.bind(progress))
.on('end', function () {
t.deepEqual(
adapt(data)
adapt(progress)
, [ { file: '/fixtures/root/index.html',
content: '<HTML>\n<BODY>\n<H1>INDEX</H1> \n</BODY>\n</HTML>\n' },
outfile: null },
{ file: '/fixtures/root/sub1/one.html',
content: '<HTML>\n<BODY>\n<H1>ONE</H1> \n</BODY>\n</HTML>\n' },
outfile: null },
{ file: '/fixtures/root/sub2/two.html',
content: '<HTML>\n<BODY>\n<H1>TWO</H1> \n</BODY>\n</HTML>\n' } ]
, 'uppercases all content and trims leading spaces'
outfile: null } ]
, 'reports progress for all files'
)
t.end()
});
})

test('\nrunning trimLeading and then toUpper transforms', function (t) {
var progress = []

// TODO: passing transforms in opposite order fails due to decodeStrings not properly applying -- no clue why
mutiny({ getOutStream: getStdOut, transforms: [ trimLeading, toUpper ] }, { root: root })
.on('error', fail.bind(t))
.on('data', [].push.bind(progress))
.on('end', function () {
t.deepEqual(
adapt(progress)
, [ { file: '/fixtures/root/index.html',
outfile: null },
{ file: '/fixtures/root/sub1/one.html',
outfile: null },
{ file: '/fixtures/root/sub2/two.html',
outfile: null } ]
, 'reports progress for all files'
)
t.end()
});
})

test('\nrunning error prone upperCase and trimLeading transforms', function (t) {
var data = []

mutiny({ root: root }, [ toUpperError, trimLeading ])
/*test('\nrunning error prone upperCase and trimLeading transforms', function (t) {
var progress = []
mutiny({ getOutStream: getStdOut, transforms: toUpperError }, { root: root })
.on('error', function (err) {
t.similar(err, /I hate to be number two/, 'propagates error');
t.end()
})
.on('data', [].push.bind(data))
.on('data', [].push.bind(progress))
.on('end', fail.bind(t));
})
})*/
12 changes: 8 additions & 4 deletions test/util/relative-entries.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,15 @@

var path = require('path');

module.exports = function relative(entries) {
function relative(file) {
return file ? file.slice(path.join(__dirname, '..').length) : file;
}

module.exports = function relativeEntries(entries) {
return entries.map(function(e) {
return {
file: e.file.slice(path.join(__dirname, '..').length)
, content: e.content
return {
file: relative(e.file)
, outfile: relative(e.outfile)
}
})
}

0 comments on commit 83ec090

Please sign in to comment.