diff --git a/README.md b/README.md index 706c1d2..5cebfa4 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ var dir = require('node-dir'); ``` #### readFiles( dir, [options], fileCallback, [finishedCallback] ) +#### readFilesStream( dir, [options], streamCallback, [finishedCallback] ) Sequentially read the content of each file in a directory, passing the contents to a callback, optionally calling a finished callback when complete. The options and finishedCallback arguments are not required. Valid options are: @@ -27,6 +28,7 @@ Valid options are: - reverse: sort files in each directory in descending order - shortName: whether to aggregate only the base filename rather than the full filepath - sort: sort files in each directory in ascending order (defaults to true) +- doneOnErr: control if done function called on error (defaults to true) A reverse sort can also be achieved by setting the sort option to 'reverse', 'desc', or 'descending' string value. @@ -45,6 +47,24 @@ dir.readFiles(__dirname, console.log('finished reading files:', files); }); +// display contents of huge files in this script's directory +dir.readFilesStream(__dirname, + function(err, stream, next) { + if (err) throw err; + var content = ''; + stream.on('data',function(buffer) { + content += buffer.toString(); + }); + stream.on('end',function() { + console.log('content:', content); + next(); + }); + }, + function(err, files){ + if (err) throw err; + console.log('finished reading files:', files); + }); + // match only filenames with a .txt extension and that don't start with a `.´ dir.readFiles(__dirname, { match: /.txt$/, diff --git a/lib/readfiles.js b/lib/readfiles.js index 2f66ee9..ea3df03 100644 --- a/lib/readfiles.js +++ b/lib/readfiles.js @@ -49,7 +49,8 @@ function readFiles(dir, options, callback, complete) { }; options = extend({ recursive: true, - encoding: 'utf8' + encoding: 'utf8', + doneOnErr: true }, options); var files = []; @@ -62,8 +63,10 @@ function readFiles(dir, options, callback, complete) { fs.readdir(dir, function(err, list) { if (err)  { - if (err.code === 'EACCES') return done(); - return done(err); + if (options.doneOnErr === true) { + if (err.code === 'EACCES') return done(); + return done(err); + } } var i = 0; @@ -78,13 +81,13 @@ function readFiles(dir, options, callback, complete) { if (!filename) return done(null, files); var file = path.join(dir, filename); fs.stat(file, function(err, stat) { - if (err) return done(err); + if (err && options.doneOnErr === true) return done(err); if (stat && stat.isDirectory()) { if (options.recursive) { if (options.matchDir && !matches(filename, options.matchDir)) return next(); if (options.excludeDir && matches(filename, options.excludeDir)) return next(); readFiles(file, options, callback, function(err, sfiles) { - if (err) return done(err); + if (err && options.doneOnErr === true) return done(err); files = files.concat(sfiles); next(); }); @@ -98,7 +101,9 @@ function readFiles(dir, options, callback, complete) { fs.readFile(file, options.encoding, function(err, data) { if (err) { if (err.code === 'EACCES') return next(); - return done(err); + if (options.doneOnErr === true) { + return done(err); + } } if (callback.length > 3) if (options.shortName) callback(null, data, filename, next); diff --git a/lib/readfilesstream.js b/lib/readfilesstream.js index 9e3a35a..7071aae 100644 --- a/lib/readfilesstream.js +++ b/lib/readfilesstream.js @@ -58,7 +58,8 @@ function readFilesStream(dir, options, callback, complete) { }; options = extend({ recursive: true, - encoding: 'utf8' + encoding: 'utf8', + doneOnErr: true }, options); var files = []; @@ -70,7 +71,12 @@ function readFilesStream(dir, options, callback, complete) { }; fs.readdir(dir, function(err, list) { - if (err) return done(err); + if (err)  { + if (options.doneOnErr === true) { + if (err.code === 'EACCES') return done(); + return done(err); + } + } var i = 0; if (options.reverse === true || @@ -84,32 +90,37 @@ function readFilesStream(dir, options, callback, complete) { if (!filename) return done(null, files); var file = path.join(dir, filename); fs.stat(file, function(err, stat) { - if (err) return done(err); + if (err && options.doneOnErr === true) return done(err); if (stat && stat.isDirectory()) { if (options.recursive) { if (options.matchDir && !matches(filename, options.matchDir)) return next(); if (options.excludeDir && matches(filename, options.excludeDir)) return next(); readFilesStream(file, options, callback, function(err, sfiles) { - if (err) return done(err); + if (err && options.doneOnErr === true) return done(err); files = files.concat(sfiles); next(); }); } else next(); - } else { + } else if (stat && stat.isFile()) { if (options.match && !matches(filename, options.match)) return next(); if (options.exclude && matches(filename, options.exclude)) return next(); if (options.filter && !options.filter(filename)) return next(); if (options.shortName) files.push(filename); else files.push(file); var stream = fs.createReadStream(file); + stream.setEncoding(options.encoding); stream.on('error',function(err) { - return done(err); + if (options.doneOnErr === true) return done(err); + next(); }); if (callback.length > 3) if (options.shortName) callback(null, stream, filename, next); else callback(null, stream, file, next); else callback(null, stream, next); } + else { + next(); + } }); })(); diff --git a/test/fixtures/testdir3/broken_link.txt b/test/fixtures/testdir3/broken_link.txt new file mode 120000 index 0000000..f69ee74 --- /dev/null +++ b/test/fixtures/testdir3/broken_link.txt @@ -0,0 +1 @@ +/broken_link \ No newline at end of file diff --git a/test/fixtures/testdir3/file1.txt b/test/fixtures/testdir3/file1.txt new file mode 100644 index 0000000..13e03c1 --- /dev/null +++ b/test/fixtures/testdir3/file1.txt @@ -0,0 +1,3 @@ +begin content of file1 +content body +end content of file1 \ No newline at end of file diff --git a/test/test.js b/test/test.js index f221e82..a4e9f6c 100644 --- a/test/test.js +++ b/test/test.js @@ -3,7 +3,8 @@ dir = require('..'), fixturesDir = path.join(__dirname, 'fixtures'), tdir = path.join(fixturesDir, 'testdir'), - tdir2 = path.join(fixturesDir, 'testdir2'); + tdir2 = path.join(fixturesDir, 'testdir2'), + tdir3 = path.join(fixturesDir, 'testdir3'); describe('readfiles method', function() { @@ -435,6 +436,33 @@ describe('readfiles method', function() { }); }); + it('should done on error', function(done) { + dir.readFiles( + tdir3, function(err, content, filename, next) { + should.not.exist(err); + var shortName = path.basename(filename).replace(new RegExp(path.extname(filename) + '$'), ''); + var expected = 'begin content of ' + shortName + '\ncontent body\nend content of ' + shortName; + content.replace(/\r/g, '').should.equal(expected); + next(); + }, function(err) { + should.exist(err); + done(); + }); + }); + + it('if given doneOnErr to false, should not done on error', function(done) { + dir.readFiles( + tdir3, { doneOnErr: false },function(err, content, filename, next) { + should.not.exist(err); + var shortName = path.basename(filename).replace(new RegExp(path.extname(filename) + '$'), ''); + var expected = 'begin content of ' + shortName + '\ncontent body\nend content of ' + shortName; + content.replace(/\r/g, '').should.equal(expected); + next(); + }, function() { + done(); + }); + }); + it('can be called with a callback in which the filename argument is omitted', function(done) { dir.readFiles( tdir, function(err, content, next) { @@ -1018,6 +1046,49 @@ describe('readfilesstream method', function() { }); }); + it('should done on error', function(done) { + dir.readFilesStream( + tdir3, function(err, stream, filename, next) { + should.not.exist(err); + var content = ''; + stream.on('data',function(buffer) { + var part = buffer.toString(); + content += part; + }); + stream.on('end',function() { + content = content.replace(/\r/g, ''); + var shortName = path.basename(filename).replace(new RegExp(path.extname(filename) + '$'), ''); + var expected = 'begin content of ' + shortName + '\ncontent body\nend content of ' + shortName; + content.should.equal(expected); + filenames.push(shortName); + next(); + }); + }, function(err) { + should.exist(err); + done(); + }); + }); + + it('if given doneOnErr to false, should not done on error', function(done) { + dir.readFilesStream( + tdir3, { doneOnErr: false },function(err, stream, filename, next) { + should.not.exist(err); + var shortName = path.basename(filename).replace(new RegExp(path.extname(filename) + '$'), ''); + var expected = 'begin content of ' + shortName + '\ncontent body\nend content of ' + shortName; + var content = ''; + stream.on('data',function(buffer) { + var part = buffer.toString(); + content += part; + }); + stream.on('end',function() { + content.replace(/\r/g, '').should.equal(expected); + next(); + }); + }, function() { + done(); + }); + }); + it('can be called with a callback in which the filename argument is omitted', function(done) { dir.readFilesStream( tdir, function(err, stream, next) {