unfluff

var p5 = require('parse5')
function uf(nm) {
    var body='', headers = '', d = new p5.SAXParser(), inHeader = false;
    d.on('text', function(text) {
        if (inHeader) {
            headers += ' ' + text;
        } else {
            body += ' ' + text;
        }
    });
    /* d.on('startTag', function(name) {
        console.log(name);
        if (name == 'dt') inHeader = true;
    })
    d.on('endTag', function(name) {
        if (name == 'dt') inHeader = false;
    }) */
    d.write(require('fs').readFileSync('app/'+nm, 'utf8'));
    return body; /* {
        body: body,
        headers: headers
    } */
}

const nodelucene = require('./build/Release/nodelucene');
var items =
    require('fs').readdirSync('app/python/library').map(function(s) { return 'python/library/'+s; }).concat(
    require('fs').readdirSync('app/clojure').filter(function(s) { return s.indexOf('.html') != -1}).map(function(s) { return 'clojure/'+s; }));

index = new nodelucene.LuceneIndex();
index.startWriting();
for (var i = 0; i < items.length; ++i) {
    index.addFile(items[i], uf(items[i]));
    console.log(i);
}
index.endWriting()