Many updates, new version

Queue state saving
Now commands writed as string instead of arrays
Removed cfg.user and very simplified code because of it
Changed project structure
Log dir now doesn't depend of log.js location

Author: sashasimkin

README.md

@@ -1,42 +1,37 @@
 About
 ===
 Git web-hook receiver written in node.js.
-Created for prevent a routine work after commit. Useful if you have a server for deploy and when using git :-)
+Created for prevent a routine work after commit. Useful if you have a server for deploy and using git :-)
 
 Requirements
 ===
 * nodejs >= v0.6.21
-* System: linux, *bsd(tested on Ubuntu)
+* System: Linux(tested on Ubuntu), *BSD(Not tested)
 
 Instalation
 ===
 ```
 git clone git://github.com/sashasimkin/hook-receiver.git
 ```
-
-Usage
-===
-1. Create file {{name}}.json with config object in dir config/ below about this).
-2. node app.js IP:PORT (IP non-required, but if you filled port only the command must looks like `node app.js :PORT`). Defaults `IP='0.0.0.0';PORT=8001`.
-3. Use `http://IP:PORT/{{name}}` as hook url in github, gitlab, etc.
-4. Set chmod 0777 on logs/ directory
+1. `cd hook-reciever/`
+2. `chmod 0777 logs/`
+2. Create file `{name}.json` with config object in directory `config/` (Below about its contents).
+3. run server `node app.js IP:PORT` (IP non-required, but if you filled port only the command must looks like `node app.js :PORT`). Defaults `IP="0.0.0.0";PORT=8001`.
+4. Use `http://IP:PORT/{name}` as hook url in github, gitlab, etc.
 
 Configuration:
 ===
 It is a json file with json object inside in directory config/. This configuration used every time when hook has been recieved.
 
 Configuration parameters:
 ===
-* `user` - System user, from which will be performed commands
-* `path` - root path for project, shell commans has been executed there
-* `commands` - Shell commands, which will be performed after recieve hook
-* `refs` - Non-required, if ref not match, any operation will not be performed. string or array of strings which be substituted to ref.match()
+* `path` - Root path for project, shell commands has been executed here
+* `commands` - Array of shell commands, which will be performed after recieve hook
+* `refs` - Commits ref to match. String or array of strings which will be substituted to ref.match()
 
 
 TODO
 ===
-* Queue state persistence
-* Get user id in system (id -u {cfg.user})
 * Variables in command definition
 * Commands string instade of array
-* cfg.require parameter with applying the cfg.commands results
+* cfg.require parameter - such as cfg.commands, but javascript files for inclusion

app.js

@@ -2,94 +2,90 @@
 var http = require("http");
 var fs = require("fs");
 var path = require("path");
-var child_process = require('child_process');
-var log = require("./lib/logging");
-var TaskManager = require("./lib/tasks/TaskManager");
-//Runtime variables
-var cfg_dir = __dirname + '/config/';
-var cfg_map = {};
+var exec = require('child_process').exec;
+var log = require("./lib/log");
+var TaskManager = require("./lib/TaskManager");
 
-//Server options
+//IP and PORT for web-server
 var run_argv = process.argv[2] ? process.argv[2].split(':') : [];
 process.env.IP = process.env.IP || run_argv[0] || '0.0.0.0';
 process.env.PORT = process.env.PORT || run_argv[1] || 8001;
 
-var processFile = function (fileName) {
-    var filePath = cfg_dir + fileName;
-    var fileExt = path.extname(fileName);
-    if (fileExt != '.json') {
-        return log('Problem with file "' + filePath + '". There must be .json file extension.', 'runtime');
-    }
-
-    fs.readFile(filePath, 'utf-8', function (err, data) {
-        var cfg = {};
-        try {
-            if (err) throw err;
+//Runtime variables
+var cfg = {
+    dir: __dirname + '/config/',
+    map: {}
+};
 
-            cfg = JSON.parse(data);
-            if ([cfg.path, cfg.user, cfg.commands].indexOf(undefined) !== -1) {
-                throw new Error('Bad config file "' + filePath + '". It need to be json object with path, user and commands keys.');
-            }
-        } catch (e) {
-            return log('Error while processing file "' + filePath + '": ' + e, 'runtime');
+/**
+ * Validate file and add to cfg map if valid
+ */
+var processFile = function (fName) {
+    var fPath = path.join(cfg.dir, fName),
+        fExt = path.extname(fName);
+    
+    if (fExt != '.json') return log('Problem with file "' + fPath + '". There must be .json file extension.', 'runtime');
+    
+    try{
+        delete require.cache[fPath];
+        var data = require(fPath);
+        
+        if (!(data.path && typeof data.commands == 'object' && data.commands.length)) {
+            throw new Error('Bad config file "' + fPath + '". It need to be json object with path and commands keys.');
         }
-        //Populate good cfg object to objects map by filename without extension
-        return cfg_map[path.basename(fileName, fileExt)] = cfg;
-    });
+        
+        fs.readdirSync(data.path);
+        
+        return cfg.map[path.basename(fName, fExt)] = data;
+    } catch(e) {
+        return log('Error while processing file "' + fPath + '": ' + e, 'runtime');
+    }
 };
 
-// Readfiles to object on server start
-fs.readdir(cfg_dir, function (wtf, files) {
-    var watchCallback = function (prev, next) {
-        processFile(files[i]);
+/**
+ * Callback for file-watchers
+ */
+var watchCallback = function(file) {
+    processFile(file);
+    return function (curr, prev) {
+        if(prev.mtime != curr.mtime) processFile(file);
     };
-
-    for (var i in files) {
+};
+//Readfiles to object on app start
+fs.readdir(cfg.dir, function (err, files) {
+    if(err) return log(err, 'startup');
+    
+    files.map(function(file, i) {
         try {
-            processFile(files[i]);
-            fs.watchFile(cfg_dir + files[i], watchCallback);
+            fs.watchFile(path.join(cfg.dir, file), watchCallback(file));
         } catch (e) {
             log(e, 'startup');
         }
-    }
-
-    //Watch for changes
-    fs.watch(cfg_dir, function (event, fileName) {
-        processFile(fileName);
     });
 });
 
+//Watch for changes in directory
+fs.watch(cfg.dir, function (event, fName) {
+    processFile(fName);
+});
+
 var runTask = function (task) {
-    task.started();
-    var cmd = task.commands.shift();
-    if (!cmd) {
-        return task.stopped();
+    task.running();
+    var command = task.commands.shift();
+    if (!command) {
+        return task.finished();
     }
-
-    var proc = child_process.spawn(cmd.shift(), cmd, task.options),
-        cmd_string = cmd.join(' '),
-        stdout = '',
-        stderror = '';
-
-    proc.stdout.on('data', function (data) {
-        stdout += data;
-    });
-    proc.stderr.on('data', function (data) {
-        stderror += data;
-    });
-    proc.on('exit', function (code, signal) {
-        //Log results of current command
-        if (stdout) log('Data from "' + cmd_string + '": ' + stdout, task.name + '.info');
-        if (stderror) log('Errors in "' + cmd_string + '": ' + stderror, task.name + '.error');
-        //Run next task, pass reference to current task
+    
+    exec(command, task.options, function(err, stdout, stderr) {
+        if (stdout) log('Data from "' + command + '": ' + stdout, task.name + '.info');
+        if (stderr) log('Error in "' + command + '": ' + stderr, task.name + '.error');
+        
         runTask(task);
     });
 };
 
-//Get initial variables, or move it inside class
-var initial = {};
-var queue = new TaskManager(initial);
-
+//Setup task manager
+var queue = new TaskManager();
 queue.run(function () {
     runTask(this);
 }, 1000);
@@ -99,9 +95,9 @@ http.createServer(function (request, response) {
     //Strip request.url, remove first slash
     request.url = request.url.slice(1);
     //Prevent favicon.ico requests
-    if (request.url == 'favicon.ico') return request.connection.destroy();
+    if (request.methom == 'GET' && request.url == 'favicon.ico') return request.connection.destroy();
 
-    if (request.method == 'POST' && cfg_map[request.url]) {
+    if (request.method == 'POST' && cfg.map[request.url]) {
         var body = '';
         request.on('data', function (data) {
             body += data;
@@ -117,42 +113,29 @@ http.createServer(function (request, response) {
             } catch (e) {
                 return log('Malformed json. Request body: ' + body, request.url + '.error');
             }
-
             //We need object copy!
-            var cfg = JSON.parse(JSON.stringify(cfg_map[request.url]));
-            var spawn_options = {
+            var task = JSON.parse(JSON.stringify(cfg.map[request.url]));
+            task.name = request.url;
+            task.options = {
                 encoding: "utf-8",
-                env: process.env
+                env: process.env,
+                cwd: task.path
             };
 
-            if (cfg.user) {
-                spawn_options.uid = cfg.user;
-            }
-
-            try {
-                fs.readdirSync(cfg.path);
-                spawn_options.cwd = cfg.path;
-            } catch (e) {
-                return log('Invalid path "' + cfg.path + '" in config "' + request.url + '"', request.url + '.error');
-            }
-
-            if (cfg.refs) {
-                var refsType = typeof cfg.refs;
+            if (task.refs) {
+                var refsType = typeof task.refs;
                 if (['string', 'object'].indexOf(refsType)) {
-                    if (refsType == 'string') cfg.refs = [cfg.refs];
+                    if (refsType == 'string') task.refs = [task.refs];
 
-                    var suitableRef = Object.keys(cfg.refs).some(function (k) {
-                        return bodyObj.ref.match(cfg.refs[k]);
+                    var suitableRef = Object.keys(task.refs).some(function (k) {
+                        return bodyObj.ref.match(task.refs[k]);
                     });
                     if (!suitableRef) return log('Ref does not fit. Aborting.', request.url + '.info');
                 }
             }
 
-            if (cfg.commands.length) {
-                cfg.name = request.url;
-                cfg.options = spawn_options;
-
-                queue.push(request.url, cfg);
+            if (task.commands.length) {
+                queue.push(request.url, task);
             } else {
                 return log('No commands to execute.', request.url + '.info');
             }

config/sample.json

@@ -3,9 +3,9 @@
     "refs": [
         "refs/heads/*"
     ],
-    "user": "shell-user",
     "commands": [
-        ["git", "fetch", "--all"],
-        ["git", "reset", "--hard", "origin/master"]
+        "git clone git://github.com/sashasimkin/hook-receiver.git",
+        "git fetch --all",
+        "git reset --hard origin/master"
     ]
 }

lib/TaskManager.js

@@ -0,0 +1,75 @@
+var os = require("os"),
+    path = require("path"),
+    fs = require("fs");
+
+exports = module.exports = function() {
+    var self = this;
+    var queue = {};
+    var state_file = path.join(os.tmpdir(), 'hook-reciever-state.json');
+    
+    //Custom not enumerable length for counting all tasks in queue
+    Object.defineProperty(queue, "length", {
+        writable: true,
+        value: 0
+    });
+    
+    self.saveState = function() {
+        if(queue.length){
+            fs.writeFile(state_file, JSON.stringify(queue), function(err) {});
+        }
+    };
+    
+    self.get = function() {
+        if(queue.length) {
+            for(var type in queue){
+                if(queue[type].tasks.length && !queue[type].running){
+                    queue.length--;
+                    return queue[type].tasks.shift();
+                }
+            }
+        }
+        //If nothing
+        return null;
+    };
+    
+    //self.push(request.url, cfg)
+    self.push = function(type, task) {
+        queue.length++;
+        if(!queue[type]) queue[type] = { running: false };
+        if(!queue[type].tasks) queue[type].tasks = [];
+        
+        task.running = function() {
+            return (queue[type].running = true);
+        };
+        task.finished = function() {
+            return (queue[type].running = false);
+        };
+        
+        queue[type].tasks.push(task);
+    };
+    
+    //Add try-catch, or not.
+    self.run = function(callback, delay) {
+        return setInterval(function() {
+            var task = self.get();
+            if(task){
+                self.saveState();
+                callback.apply(task);
+            }
+        }, delay);
+    };
+    
+    self.stop = function(id){
+        return clearInterval(id);
+    };
+    
+    try{
+        //It maybe alredy included, I dont know how. In any case does not prevent
+        delete require.cache[state_file];
+        queue = require(state_file);
+        //Mark all tasks as stopped, if we fall when some task executing
+        for(var k in queue) queue[k].running = false;
+    } catch(e) {}
+    
+    return self;
+};

lib/logging/log.js lib/log.js

@@ -2,7 +2,7 @@ var fs = require("fs");
 var path = require("path");
 
 var dbg = true;
-var logs_dir = __dirname + '/../../logs';
+var logs_dir = path.join(process.cwd(), 'logs');
 
 Date.prototype.toLocaleFormat = function(format) {
     var f = {y : this.getYear() + 1900,m : this.getMonth() + 1,d : this.getDate(),H : this.getHours(),M : this.getMinutes(),s : this.getSeconds()};