solr job added

webcrawler/job/solr.js

@@ -4,6 +4,13 @@ var solr = require('solr-client');
 exports = module.exports = Solr;
 
 function Solr(options) {
+	for( i in options.rules ) {
+		if( options.rules[ i ].filter != undefined ) {
+			options.rules[ i ].filter.pattern = new RegExp( options.rules[ i ].filter.pattern, 'g' );
+		}
+
+	}
+
 	this.options = options;
 }
 
@@ -19,13 +26,49 @@ Solr.prototype.eachRule = function(callback) {
 	}
 }
 
+Solr.prototype.save = function(doc, callback) {
+	var client = solr.createClient();
+
+	client.add(doc, function(err, obj) {
+		if( err ) {
+			console.log( " --- Failed inserting document to Solr --- " );
+			console.log( err );
+			console.log( " --------- " );
+			console.log( doc );
+			console.log( " --------- " );
+		}
+		else {
+			client.commit(function(err, res) {
+				if( err ) {
+					console.log( " --- Failed commiting document to Solr --- " );
+					console.log( err );
+					console.log( " --------- " );
+					console.log( doc );
+					console.log( " --------- " );
+				}
+			});
+		}
+
+		callback();
+	});
+}
+
+Solr.prototype.applyFilter = function( str, rule ) {
+	if( rule.filter == undefined ) {
+		return str;
+	}
+
+	return str.replace( rule.filter.pattern, rule.filter.replacement );
+}
+
 Solr.prototype.execute = function(callback, data, env) {
 	if( env.res.headers['content-type'] == undefined || ! env.res.headers['content-type'].match( /^text\/html/) ) {
 		return callback();
 	}
 
 	var $ = cheerio.load( data ),
 		self = this,
+		saveDoc = false,
 		doc = {
 			id: env.task.href
 		};
@@ -39,6 +82,8 @@ Solr.prototype.execute = function(callback, data, env) {
 				var content = $(this).attr( rule.attribute );
 			}
 
+			content = self.applyFilter( content, rule );
+
 			if( doc[ rule.field ] == undefined ) {
 				doc[ rule.field ] = content;
 			}
@@ -48,33 +93,17 @@ Solr.prototype.execute = function(callback, data, env) {
 			else {
 				doc[ rule.field ].push( content );
 			}
+
+			saveDoc = true;
 		});
 	});
 
-	var client = solr.createClient();
-
-	client.add(doc, function(err, obj) {
-		if( err ) {
-			console.log( " --- Failed inserting document to Solr --- " );
-			console.log( err );
-			console.log( " --------- " );
-			console.log( doc );
-			console.log( " --------- " );
-		}
-		else {
-			client.commit(function(err, res) {
-				if( err ) {
-					console.log( " --- Failed commiting document to Solr --- " );
-					console.log( err );
-					console.log( " --------- " );
-					console.log( doc );
-					console.log( " --------- " );
-				}
-			});
-		}
-
+	if( saveDoc ) {
+		this.save( doc, callback );
+	}
+	else {
 		callback();
-	});
+	}
 
 }