From 6d58d268764e0a5458257fec16d04e0a83e6ae26 Mon Sep 17 00:00:00 2001 From: s asfd Date: Mon, 17 Aug 2015 09:54:40 -0700 Subject: [PATCH] Caching of files on agent side, Better logs with download, search. Compile only when there are changes. New sencha. --- agent/.gitignore | 3 +- agent/app.js | 13 +- agent/routes/fileupload.js | 320 +++++++++++++++++++++++++++----- app.js | 7 +- index.html | 2 +- package.json | 2 +- public/controller/Executions.js | 20 +- public/view/ResultsView.js | 57 +++++- public/view/Variables.js | 4 +- routes/compile.js | 6 +- routes/executionengine.js | 268 ++++++++++++++++++++++---- routes/results.js | 16 ++ 12 files changed, 604 insertions(+), 114 deletions(-) diff --git a/agent/.gitignore b/agent/.gitignore index e2cb85c1..5bd9d699 100644 --- a/agent/.gitignore +++ b/agent/.gitignore @@ -3,4 +3,5 @@ bin executionfiles Java agent.pid -agent.vnc.pid \ No newline at end of file +agent.vnc.pid +cache \ No newline at end of file diff --git a/agent/app.js b/agent/app.js index ea4a2c0b..c39e1376 100644 --- a/agent/app.js +++ b/agent/app.js @@ -10,6 +10,7 @@ var common = require('./common'); var path = require('path'); //var idesync = require('./routes/idesync'); +process.setMaxListeners(0); var app = express(); process.env.TMPDIR = path.resolve(__dirname,"../logs"); @@ -23,29 +24,27 @@ app.configure(function(){ //app.use(express.cookieParser()); app.use(express.methodOverride()); app.use(app.router); + app.use(express.timeout(300000)); //app.use(express.bodyParser({ keepExtensions: true, uploadDir: '/bin' })); }); app.post('/command',command.Post); app.post('/update',update.Post); app.post('/fileupload',fileupload.Post); +app.post('/matchfile',fileupload.MatchFile); app.post('/uploadfiles',uploadfiles.uploadFiles); app.post('/recordimage',imageautomation.recordImage); app.post('/startrecording',recorder.record); -app.configure('development', function(){ - app.use(express.errorHandler({ dumpExceptions: true, showStack: true })); -}); - -app.configure('production', function(){ - app.use(express.errorHandler()); -}); +app.use(express.errorHandler({ dumpExceptions: true, showStack: true })); +app.use(express.errorHandler()); common.initLogger("agent"); common.parseConfig(function(){ app.listen(common.Config.AgentPort, function(){ if(common.Config.CloudAgent !== "true"){ heartbeat.startHeartBeat(common.Config.AppServerIPHost,common.Config.AppServerPort,common.Config.AgentPort,common.Config.AgentVNCPort,common.Config.AgentVersion,common.Config.OS); } + common.logger.info('Agent Started.'); command.cleanUp(); //console.log("Express server listening on port %d in %s mode", app.address().port, app.settings.env); }); diff --git a/agent/routes/fileupload.js b/agent/routes/fileupload.js index f2feacc4..e23c65dc 100644 --- a/agent/routes/fileupload.js +++ b/agent/routes/fileupload.js @@ -4,14 +4,64 @@ var walk = require('walk'); var common = require('../common'); var spawn = require('child_process').spawn; var AdmZip = require('adm-zip'); +var cacheDir = path.resolve(__dirname,"../cache")+"/"; +var fileSync = {}; -exports.Post = function(req, res){ - var tmp_path = req.files.file.path; - var target_path = path.resolve(__dirname,"../"+req.files.file.name); - common.logger.info(target_path); +exports.MatchFile = function(req,res){ + var dirName = getUniqueDirName(req.body.dest); + //extract file name + var fileName = req.body.dest.split("/"); + fileName = fileName[fileName.length-1]; + var i = 1; + var matchFile = function(){ + if(i>5){ + res.send('{"error":null,"success":true,"matched":false}'); + return; + } + var fullPath = cacheDir + dirName + "_cache/v"+i+"/" + fileName; + if(req.body.file.indexOf("pythonSources.zip") != -1){ + res.send('{"error":null,"success":true,"matched":false}'); + return; + } + if(fs.existsSync(fullPath) == true){ + if(fs.existsSync(cacheDir + dirName + "_cache/v"+i+"/md5.txt") == false){ + res.send('{"error":null,"success":true,"matched":false}'); + return; + } + + if(fs.existsSync(cacheDir + dirName + "_cache/v"+i+"/"+fileName) == false){ + res.send('{"error":null,"success":true,"matched":false}'); + return; + } + fs.readFile(cacheDir + dirName + "_cache/v"+i+"/md5.txt", function(err, buf) { + //var md5Value = md5(buf); + if(buf.toString() == req.body.md5){ + var target_path = path.resolve(__dirname,"../"+req.body.dest); + console.log("COPYING OVER:"+target_path); + CreateParentDirs(req.body.dest,function(){ + copyFile(fullPath,target_path,function(){ + res.send('{"error":null,"success":true,"matched":true}'); + }) + }); + i = 7; + } + else{ + i++; + matchFile(); + } + }); + } + else{ + console.log("FALSE I IS:"+fullPath); + res.send('{"error":null,"success":true,"matched":false}'); + } + }; + matchFile(); +}; - if (req.files.file.name.indexOf("/") != -1){ - var dirs = req.files.file.name.slice(0,req.files.file.name.lastIndexOf("/")); +function CreateParentDirs(file,callback){ + if (file.indexOf("/") != -1){ + var dirs = file.slice(0,file.lastIndexOf("/")); var parent = ""; dirs.split("/").forEach(function(dir){ if (fs.existsSync(path.resolve(__dirname,"../"+parent+dir)) == false){ @@ -20,55 +70,148 @@ exports.Post = function(req, res){ parent = parent + dir+"/"; }); } - try{ - fs.rename(tmp_path, target_path, function(err) { - if (err){ - res.send('{error:"'+err+'"}'); - common.logger.error("rename ERROR:"+err); - fs.unlink(tmp_path); - return; - } - if(req.files.file.name.indexOf("pythonLibs.zip") != -1){ - var extractTo = path.resolve(__dirname,"../")+"/"+req.files.file.name.substring(0,req.files.file.name.lastIndexOf("/")); - var zip = new AdmZip(target_path); - zip.extractAllTo(extractTo, true); - res.send("{error:null,success:true}"); - //var unzip = spawn(path.resolve(__dirname,'../../vendor/Java/bin/jar'),['xf','pythonLibs.zip'],{cwd: extractTo,timeout:300000}); - } - else if(req.files.file.name.indexOf("pythonSources.zip") != -1){ - var extractTo = path.resolve(__dirname,"../")+"/"+req.files.file.name.substring(0,req.files.file.name.lastIndexOf("/")); - var pythonFileName; - if(require('os').platform() == "win32"){ - pythonFileName = "python.exe" - } - else{ - pythonFileName = "python" + callback(); +} + +function StoreMD5(file,md5File){ + if (file in fileSync){ + if(fileSync[file].close){ + fileSync[file].destroy(); + } + } + var md5sum = require("crypto").createHash('md5'); + var s = fs.ReadStream(file); + + s.on('data',function(d){ + md5sum.update(d); + }); + + s.on("error",function(){ + this.close(); + delete fileSync[file]; + }); + + s.on('close',function(){ + delete fileSync[file]; + var d = md5sum.digest('hex'); + fs.writeFile(md5File, d.toString()); + }); +} + +function getUniqueDirName(fullPath,callback){ + var names = fullPath.split("/"); + var name = ""; + for(var i =2;i 5){ + //if(fs.existsSync(cacheDir + fileName + "/v1/"+fileName))fs.unlinkSync(cacheDir + fileName + "/v1/"+fileName); + var files = fs.readdirSync(cacheDir + dirName+"_cache") + .map(function(v) { + return { name:v, + time:fs.statSync(cacheDir + dirName+ "_cache/" + v).mtime.getTime() + }; + }) + .sort(function(a, b) { return a.time - b.time; }) + .map(function(v) { return v.name; }); + var oldestDir = cacheDir + dirName + "_cache/" + files[0]; + try{ + if(fs.existsSync(oldestDir+"/" + fileName)) {fs.unlinkSync(oldestDir+"/" + fileName);} + if(fs.existsSync(oldestDir+"/md5.txt")) {fs.unlinkSync(oldestDir+"/md5.txt");} + if(fs.existsSync(oldestDir)) {fs.rmdirSync(oldestDir);} + fs.mkdirSync(oldestDir); + copyFile(target_path,oldestDir+"/"+fileName,function(){ + StoreMD5(oldestDir+"/"+fileName,oldestDir + "/md5.txt"); + }); + } + catch(e){} + } + +} + +exports.Post = function(req, res){ + var tmp_path = req.files.file.path; + var target_path = path.resolve(__dirname,"../"+req.files.file.name); + common.logger.info(target_path); + CreateParentDirs(req.files.file.name,function(){ + try{ + fs.rename(tmp_path, target_path, function(err) { + if (err){ + res.send('{error:"'+err+'"}'); + common.logger.error("rename ERROR:"+err); + fs.unlink(tmp_path); + return; } - copyFile(path.resolve(__dirname,'../../vendor/Python')+"/"+pythonFileName,path.resolve(extractTo,"../")+"/"+pythonFileName,function(){ - extractTo = path.resolve(extractTo,"../")+"/src/"; + if(req.files.file.name.indexOf("pythonLibs.zip") != -1){ + var extractTo = path.resolve(__dirname,"../")+"/"+req.files.file.name.substring(0,req.files.file.name.lastIndexOf("/")); var zip = new AdmZip(target_path); zip.extractAllTo(extractTo, true); res.send("{error:null,success:true}"); + SaveToCache(req.files.file.name,target_path); + //var unzip = spawn(path.resolve(__dirname,'../../vendor/Java/bin/jar'),['xf','pythonLibs.zip'],{cwd: extractTo,timeout:300000}); + } + else if(req.files.file.name.indexOf("pythonSources.zip") != -1){ + var extractTo = path.resolve(__dirname,"../")+"/"+req.files.file.name.substring(0,req.files.file.name.lastIndexOf("/")); + var pythonFileName; + if(require('os').platform() == "win32"){ + pythonFileName = "python.exe" + } + else{ + pythonFileName = "python" + } + copyFile(path.resolve(__dirname,'../../vendor/Python')+"/"+pythonFileName,path.resolve(extractTo,"../")+"/"+pythonFileName,function(){ + extractTo = path.resolve(extractTo,"../")+"/src/"; + var zip = new AdmZip(target_path); + zip.extractAllTo(extractTo, true); + res.send("{error:null,success:true}"); - //fs.mkdir(extractTo,function(){ - // var unzip = spawn(path.resolve(__dirname,'../../vendor/Java/bin/jar'),['xf',target_path],{cwd: extractTo,timeout:300000}); - }); - } - else if(req.files.file.name.indexOf("RedwoodHQAutomation.dll") != -1 ){ - var extractTo = path.resolve(__dirname,"../")+"/"+req.files.file.name.substring(0,req.files.file.name.lastIndexOf("/")); - copyFile(path.resolve(__dirname,'../lib')+"/CSharpLauncher.exe",path.resolve(extractTo,"../")+"/lib/CSharpLauncher.exe",function(){ + //fs.mkdir(extractTo,function(){ + // var unzip = spawn(path.resolve(__dirname,'../../vendor/Java/bin/jar'),['xf',target_path],{cwd: extractTo,timeout:300000}); + }); + } + else if(req.files.file.name.indexOf("RedwoodHQAutomation.dll") != -1 ){ + var extractTo = path.resolve(__dirname,"../")+"/"+req.files.file.name.substring(0,req.files.file.name.lastIndexOf("/")); + copyFile(path.resolve(__dirname,'../lib')+"/CSharpLauncher.exe",path.resolve(extractTo,"../")+"/lib/CSharpLauncher.exe",function(){ + res.send("{error:null,success:true}"); + }); + SaveToCache(req.files.file.name,target_path); + } + else{ + //create cache res.send("{error:null,success:true}"); - }) - } - else{ - res.send("{error:null,success:true}"); - } - }); - } - catch(exception){ - res.send('{error:"'+exception+'"}'); - common.logger.error("EXCEPTION while renaming file:"+target_path+" "+exception); - } + SaveToCache(req.files.file.name,target_path); + } + }); + } + catch(exception){ + res.send('{error:"'+exception+'"}'); + common.logger.error("EXCEPTION while renaming file:"+target_path+" "+exception); + } + }); + //console.log(tmp_path); //console.log(target_path); @@ -76,20 +219,95 @@ exports.Post = function(req, res){ function copyFile(source, target, cb) { var cbCalled = false; + if (source in fileSync){ + if(fileSync[source].close){ + fileSync[source].destroy(); + } + } + else{ + fileSync[source] = true; + } + + var rd = fs.createReadStream(source); + fileSync[source] = rd; + var wr = fs.createWriteStream(target); + fileSync[target] = wr; + + wr.on("error", function(err) { + if(source in fileSync && fileSync[source].close){ + fileSync[source].destroy(); + } + delete fileSync[source]; + delete fileSync[target]; + this.close(); + rd.destroy.bind(rd); + done(err); + }); + wr.on("close", function(ex) { + done(); + rd.destroy.bind(rd); + }); + + rd.on("close",function(){ + if(source in fileSync && fileSync[source].close){ + fileSync[source].destroy(); + fileSync[target].destroy(); + } + delete fileSync[source]; + delete fileSync[target]; + }); + + rd.on("error",function(e){ + if(source in fileSync && fileSync[source].close){ + fileSync[source].destroy(); + fileSync[target].destroy(); + } + delete fileSync[source]; + delete fileSync[target]; + this.end(); + done(err); + }).pipe(wr, { end: true }); + + function done(err) { + if(fileSync[source]) {fileSync[source].destroy();delete fileSync[source]} + if(fileSync[target]) {fileSync[target].destroy();delete fileSync[target]} + if (!cbCalled) { + cb(err); + cbCalled = true; + } + } +} + +function copyFile_old(source, target, cb) { + if(fullPath in fileSync){ + if(fileSync[file].close){ + fileSync[file].destroy(); + } + } + var cbCalled = false; var rd = fs.createReadStream(source); rd.on("error", function(err) { + this.close(); + this.destroy(); done(err); }); var wr = fs.createWriteStream(target); + wr.on("error", function(err) { - wr.destroy(); + this.close(); done(err); }); wr.on("close", function(ex) { done(); }); - rd.pipe(wr); + + rd.on("error",function(e){ + this.end(); + done(err); + }).pipe(wr, { end: true }); + + //rd.pipe(wr); function done(err) { if (!cbCalled) { diff --git a/app.js b/app.js index 20bcfea2..b54be8dd 100644 --- a/app.js +++ b/app.js @@ -47,11 +47,12 @@ var express = require('express') , recorder = require('./routes/recorder') , license = require('./routes/license') , actionHistory = require('./routes/actionHistory') + , versionControl = require('./routes/versionControl') , testcaseHistory = require('./routes/testcaseHistory'); //var app = express.createServer(); -process.setMaxListeners(300); +process.setMaxListeners(0); var app = express(); process.env.TMPDIR = __dirname + '/logs'; process.env.TMP = __dirname + '/logs'; @@ -82,7 +83,6 @@ process.env.TEMP = __dirname + '/logs'; //}); //DB - // Routes app.post('/login',auth.logIn,auth.logInSucess); app.get('/login',auth.loginPage); @@ -90,6 +90,9 @@ app.get('/login',auth.loginPage); app.post('/license',auth.auth,license.licensePost); app.get('/license',auth.auth,license.licenseGet); +//versioncontrol +app.post('/versioncontrolhistory',auth.auth,versionControl.getLocalVersionHistory); + //emailsettings app.post('/emailsettings',auth.auth,emailsettings.Post); app.get('/emailsettings',auth.auth,emailsettings.Get); diff --git a/index.html b/index.html index 4f9217b8..e0902ac6 100644 --- a/index.html +++ b/index.html @@ -6,7 +6,7 @@ - + diff --git a/package.json b/package.json index c5c606ba..8f123f90 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ , "walk" : "2.2.1" , "optimist" : "0.5.2" , "forever-monitor" : "1.1.0" - , "xml-writer" : "1.4.0" + , "xml-writer" : ">=1.4.0" , "nodemailer" : "0.5.2" , "winston" : "0.7.2" , "multiparty" : "3.1.1" diff --git a/public/controller/Executions.js b/public/controller/Executions.js index 287547ba..56fa3659 100644 --- a/public/controller/Executions.js +++ b/public/controller/Executions.js @@ -312,7 +312,25 @@ Ext.define("Redwood.controller.Executions", { var id = execution._id; delete execution._id; execution.status = "Ready To Run"; - executionView.dataRecord = this.getStore('Executions').add(execution)[0]; + //execution.id = Ext.id(); + var newRecord = new Redwood.model.Executions({}); + try{ + this.getStore('Executions').add(newRecord); + } + catch(e){ + Ext.Msg.alert('Error', "Unable to Save execution "+ e.message); + executionView.dataRecord = null; + return; + } + + executionView.dataRecord = newRecord; + + for(var propt in execution){ + executionView.dataRecord.set(propt,execution[propt]) + } + //executionView.dataRecord = this.getStore('Executions').add(execution)[0]; + //executionView.dataRecord = this.getStore('Executions').data.add(execution); + //executionView.dataRecord._id = id; executionView.dataRecord.set("_id",id); executionView.dataRecord.phantom = true; window.history.replaceState("", "", '/index.html?execution='+id+"&project="+Ext.util.Cookies.get('project')); diff --git a/public/view/ResultsView.js b/public/view/ResultsView.js index 9c8f5e86..f6a09d77 100644 --- a/public/view/ResultsView.js +++ b/public/view/ResultsView.js @@ -2,7 +2,7 @@ Ext.define('Redwood.view.ResultsView', { extend: 'Ext.panel.Panel', alias: 'widget.resultsview', - overflowY: 'auto', + //overflowY: 'auto', bodyPadding: 5, dataRecord: null, viewType: "Results", @@ -497,13 +497,62 @@ Ext.define('Redwood.view.ResultsView', { itemId:"executionLogs", selType: 'rowmodel', height:500, - overflowY: 'auto', viewConfig: { markDirty: false, enableTextSelection: true }, - plugins: [ - "bufferedrenderer"], + tbar:{ + xtype: 'toolbar', + dock: 'top', + items: [ + { + width: 400, + fieldLabel: 'Search', + labelWidth: 50, + xtype: 'searchfield', + paramNames: ["message"], + store: me.logStore + },"", + { + icon: 'images/down.png', + hidden:false, + tooltip:"Download Logs.", + handler: function(){ + + var link = document.createElement("a"); + var records = ""; + me.logStore.each(function(record){ + records = records + record.get("actionName") + "\t" + record.get("date")+"\t"+record.get("message") + "\r\n" + }); + + if(Ext.isIE){ + var blob = new Blob([records],{ + type: "text/csv;charset=utf-8;" + }); + + navigator.msSaveBlob(blob, "log.txt"); + } + else{ + var a = window.document.createElement('a'); + a.href = window.URL.createObjectURL(new Blob([records], {type: 'text/csv'})); + a.download = 'log.txt'; + + // Append anchor to body. + document.body.appendChild(a); + a.click(); + + // Remove anchor from body + document.body.removeChild(a); + } + } + } + ] + }, + plugins: [{ + ptype:"bufferedrenderer", + trailingBufferZone:200, + leadingBufferZone:200 + }], columns:[ { header: 'Action Name', diff --git a/public/view/Variables.js b/public/view/Variables.js index 31e3cd4e..4f3622d9 100644 --- a/public/view/Variables.js +++ b/public/view/Variables.js @@ -129,10 +129,10 @@ Ext.define('Redwood.view.Variables', { displayNULLOption:true, listeners:{ validitychange: function(field,isValid){ - if(variablesEditor.rowEditor.editor) variablesEditor.rowEditor.editor.onFieldChange(); + //if(variablesEditor.rowEditor.editor) variablesEditor.rowEditor.editor.onFieldChange(); }, focus: function(){ - this.selectText(); + //this.selectText(); } }, getDisplayValue: function() { diff --git a/routes/compile.js b/routes/compile.js index c36d935e..c572deb2 100644 --- a/routes/compile.js +++ b/routes/compile.js @@ -11,9 +11,9 @@ exports.operation = function(msg, id,callback,onFinish){ if(compileProcs[id].pythonProc) compileProcs[id].pythonProc.kill(); if(compileProcs[id].csharpProc) compileProcs[id].csharpProc.kill(); } - if(!msg.java) msg.java = true; - if(!msg.python) msg.python = true; - if(!msg.csharp) msg.csharp = true; + //if(!msg.java) msg.java = true; + //if(!msg.python) msg.python = true; + //if(!msg.csharp) msg.csharp = true; var buildDir = path.resolve(__dirname,"../public/automationscripts/"+msg.project)+"/"+msg.username; compileProcs[id] = {}; diff --git a/routes/executionengine.js b/routes/executionengine.js index 76b4a737..ab88dd96 100644 --- a/routes/executionengine.js +++ b/routes/executionengine.js @@ -18,6 +18,7 @@ var os = require('os'); var archiver = require('archiver'); var db; var compilations = {}; +var fileSync = {}; exports.stopexecutionPost = function(req, res){ var execution = executions[req.body.executionID]; @@ -392,6 +393,7 @@ function applyMultiThreading(executionID,callback){ } newMachine.threadID = i+startThread-1; common.logger.info(newMachine); + if(!executions[executionID]) return; executions[executionID].machines.push(newMachine); sendAgentCommand(newMachine.host,newMachine.port,{command:"start launcher",executionID:executionID,threadID:newMachine.threadID},3,function(err){ newMachineCount++; @@ -1462,70 +1464,200 @@ function syncFilesWithAgent(agentHost,port,rootPath,destDir,callback){ } files.push({file:root+"/"+fileStats.name,dest:dest}); + sendFileToAgent(root+"/"+fileStats.name,dest,agentHost,port,0,function(error){ + fileCount++; + if(fileCount === files.length){ + callback(); + } + }) }); walker.on("end",function(){ - sendFiles() + //sendFiles() }); } -function sendFileToAgent(file,dest,agentHost,port,retryCount,callback){ - var stat = fs.statSync(file); - - var readStream = fs.createReadStream(file); - var boundary = '--------------------------'; - for (var i = 0; i < 24; i++) { - boundary += Math.floor(Math.random() * 10).toString(16); - } - - var message = '------' + boundary + '\r\n' - // use your file's mime type here, if known - + 'Content-Disposition: form-data; name="file"; filename="'+dest+'"\r\n' - + 'Content-Type: application/octet-stream\r\n' - // "name" is the name of the form field - // "filename" is the name of the original file - + 'Content-Transfer-Encoding: binary\r\n\r\n'; - - +//if agent already got the file in cache don't copy +function matchFileWithAgent(file,dest,agentHost,port,retryCount,callback){ var options = { hostname: agentHost, port: port, - path: '/fileupload', + path: '/matchfile', method: 'POST', headers: { - //'Content-Type': 'text/plain'//, - 'Content-Type': 'multipart/form-data; boundary=----'+boundary, - //'Content-Disposition': 'form-data; name="file"; filename="ProjectName.jar"', - //'Content-Length': 3360 - //'Content-Length': stat.size + message.length + 30 + boundary.length - 'Content-Length': stat.size + message.length + boundary.length + 14 + 'Content-Type': 'application/json' } }; var req = http.request(options, function(res) { - //res.setEncoding('utf8'); + res.setEncoding('utf8'); res.on('data', function (chunk) { - if (callback) callback(); + try{ + var msg = JSON.parse(chunk); + } + catch(err){ + if (callback) callback(err); + } + + if((msg )&&(msg.error != null)){ + if (callback) callback(msg.error); + } + else if (msg){ + if(callback) callback(msg); + } + else{ + if(callback) callback(); + } }); }); - req.on('error', function(e) { if(retryCount <= 0){ - if (callback) callback({error:'sendFileToAgent problem with request: ' + e.message+ ' file:'+file}); - common.logger.error('sendFileToAgent problem with request: ' + e.message+ ' file:'+file); + if (callback) callback("Unable to connect to machine: "+agentHost + " error: " + e.message); + common.logger.error('matchFileWithAgent problem with request: ' + e.message+ ' '); } else{ retryCount--; - setTimeout(sendFileToAgent(file,dest,agentHost,port,retryCount,callback),1000) + setTimeout(matchFileWithAgent(file,dest,agentHost,port,retryCount,callback),1000) } }); + //fs.readFile(file, function(err, buf) { + // write data to request body + //req.write(JSON.stringify({dest:dest,file:file,md5:md5(buf)})); + var md5sum = require("crypto").createHash('md5'); + var s = fs.ReadStream(file); + fileSync[file] = s; + s.on('data',function(d){ + md5sum.update(d); + }); + + s.on('error',function(err){ + s.destroy.bind(s); + s.end(); + }); - req.write(message); - readStream.pipe(req, { end: false }); - readStream.on("end", function(){ - req.end('\r\n------' + boundary + '--\r\n'); + s.on('close',function(){ + var d = md5sum.digest('hex'); + this.destroy(); + req.write(JSON.stringify({dest:dest,file:file,md5:d.toString()})); + req.end(); + //md5sum.end(); }); + //}); +} + +function sendFileToAgent(file,dest,agentHost,port,retryCount,callback){ + if (file in fileSync){ + if(fileSync[file].close){ + fileSync[file].destroy(); + delete fileSync[file]; + } + } + fileSync[file] = true; + matchFileWithAgent(file,dest,agentHost,port,0,function(response){ + if(response && response.matched == true){ + if(file in fileSync && fileSync[file].close){ + fileSync[file].destroy(); + } + delete fileSync[file]; + if (callback) callback(); + return; + } + var stat = fs.statSync(file); + + var readStream = fs.createReadStream(file); + fileSync[file] = readStream; + var boundary = '--------------------------'; + for (var i = 0; i < 24; i++) { + boundary += Math.floor(Math.random() * 10).toString(16); + } + + var message = '------' + boundary + '\r\n' + // use your file's mime type here, if known + + 'Content-Disposition: form-data; name="file"; filename="'+dest+'"\r\n' + + 'Content-Type: application/octet-stream\r\n' + // "name" is the name of the form field + // "filename" is the name of the original file + + 'Content-Transfer-Encoding: binary\r\n\r\n'; + + + + var options = { + hostname: agentHost, + port: port, + path: '/fileupload', + method: 'POST', + headers: { + //'Content-Type': 'text/plain'//, + 'Content-Type': 'multipart/form-data; boundary=----'+boundary, + //'Content-Disposition': 'form-data; name="file"; filename="ProjectName.jar"', + //'Content-Length': 3360 + //'Content-Length': stat.size + message.length + 30 + boundary.length + 'Content-Length': stat.size + message.length + boundary.length + 14 + } + }; + + var req = http.request(options, function(res) { + //res.setEncoding('utf8'); + res.on('data', function (chunk) { + if (file in fileSync){ + if(fileSync[file].close){ + fileSync[file].destroy(); + delete fileSync[file]; + } + } + if (callback) callback(); + }); + res.on('close', function(){ + if(file in fileSync && fileSync[file].close){ + fileSync[file].destroy(); + } + delete fileSync[file]; + readStream.destroy.bind(readStream); + }); + }); + + var handleError = function(e){ + if(file in fileSync && fileSync[file].close){ + fileSync[file].destroy(); + } + delete fileSync[file]; + if(retryCount <= 0){ + if (callback) callback({error:'sendFileToAgent problem with request: ' + e.message+ ' file:'+file}); + common.logger.error('sendFileToAgent problem with request: ' + e.message+ ' file:'+file); + } + else{ + retryCount--; + setTimeout(sendFileToAgent(file,dest,agentHost,port,retryCount,callback),1000) + } + }; + req.on('error', function(e) { + handleError(e); + this.end(); + }); + + req.write(message); + //readStream.pipe(req, { end: false }); + readStream.on("error",function(e){ + this.end(); + handleError(e); + }).pipe(req, { end: false }); + readStream.on("end", function(){ + try{ + req.end('\r\n------' + boundary + '--\r\n'); + } + catch(e){ + //req.end(); + readStream.end(); + } + }); + readStream.on('close',function(){ + if(file in fileSync && fileSync[file].close){ + fileSync[file].destroy(); + } + delete fileSync[file]; + }) + }) } exports.sendAgentCommand = function(agentHost,port,command,retryCount,callback){sendAgentCommand(agentHost,port,command,retryCount,callback)}; @@ -2132,6 +2264,15 @@ function GetTestCaseDetails(testcaseID,executionID,callback){ cb(); return; } + for(var iTCParamCount=0;iTCParamCount