fix xome bugs

conf.example.json

@@ -1,14 +1,16 @@
 {
-	"workers": 5,
-	"seedUrl": "http://stackoverflow.com/",
+	"workers": 2,
+	"seedUrl": "http://127.0.0.1:5984/_utils/docs/",
 	"database": {
-		"name": "web-crawler-so",
+		"name": "web-crawler-example2",
 		"rebuild": true
 	},
 	"jobs": [
 		{ "name": "logger" },
 		{ "name": "saver" },
-		{ "name": "driller", "domainRestriction": "stackoverflow.com" },
+		{ "name": "driller",
+			"domainRestriction": "127.0.0.1"
+		 },
 		{ "name": "scheduler" }
 	]
 }

crawler.js

@@ -1,34 +1,15 @@
-/*
-var heapdump = require('heapdump')
-var http = require('http');
-http.createServer(function (req, res) {
-  heapdump.writeSnapshot();
-  res.writeHead(200, {'Content-Type': 'text/plain'});
-  res.end("Dumped!");
-}).listen(9999);
-*/
-
 var argv = require('optimist').argv;
 var fs = require('fs');
 
 var agent = require('./webcrawler/agent');
 var Config = require( './webcrawler/config' );
 
-/*
-setInterval(function () {
-    if (typeof gc === 'function') {
-        gc();
-    }
-    console.log( process.memoryUsage());
-}, 60000);
-*/
-
 console.log( "" );
 
 if( argv._[0] == undefined ) {
   console.log( "Usage: node crawler.js [config.json]" );
   console.log( "" );
-  console.log( "Example: node crawler.js [conf.example.json]" );
+  console.log( "Example: node crawler.js conf.example.json" );
   console.log( "" );
   process.exit(1);
 }

package.json

@@ -3,10 +3,9 @@
   "version": "0.0.1",
   "private": true,
   "scripts": {
-    "start": "node monitor.js"
+    "start": "node crawler.js conf.example.json"
   },
   "dependencies": {
-    "express": "3.4.2",
     "jade": "*",
     "cheerio": "*",
     "nano": "*",

webcrawler/agent.js

@@ -8,7 +8,6 @@ var cheerio = require('cheerio');
 
 agent.initFromConfig = function( config ) {
 	var async = require( 'async' );
-	var job = require('./job');
 
 	agent.init({ workers: config.getWorkers() });
 

webcrawler/job/driller.js

@@ -17,7 +17,8 @@ function Driller(options) {
 		filters: [],
 		patterns: [],
 		verbose: false,
-		maxDepth: false
+		maxDepth: false,
+		storeSource: true
 	};
 
 	this.initOptions( options );
@@ -73,6 +74,10 @@ Driller.prototype.execute = function(callback, $, env) {
 }
 
 Driller.prototype.addSourceToWebDoc = function( url, urls, source ) {
+	if( ! this.options.storeSource ) {
+		return;
+	}
+
 	if( urls[ url ] != undefined ) { 
 		return;
 	}
@@ -124,7 +129,7 @@ Driller.prototype.setNormalasiers = function( normalisers ) {
 
 Driller.prototype.setDomainRestriction = function( domain ) {
 	var domain = domain.replace( /\./, '\.' );
-	domain = '^http[s]{0,1}:\/\/(([^\/]+?\\.)|())' + domain + '(\/|$)'; 
+	domain = '^http[s]{0,1}:\/\/(([^\/]+?\\.)|())' + domain + '(\/|:|$)'; 
 	this.options[ "domain" ] = new RegExp( domain, 'i' );
 }
 

webcrawler/storage/couchdb.js

@@ -11,6 +11,8 @@ couchdb.init = function(options) {
 
 	this.nano = require('nano')( this.options.host );
 	this.db = this.nano.use( this.options.dbname );
+
+	couchdb._nano = this.nano;
 }
 
 couchdb.getDB = function() {