trying other cover when the cover is corrupt or the page is changed a…

…nd capacity is reduced' havent' tested yet

doc/how_to_debug_with_emacs.txt

@@ -0,0 +1,15 @@
+* First run a socks server:
+  ssh -ND 5002 localhost.
+
+* Both client and server:
+  open two emacs
+
+  in one for server:
+
+  open ../gdb_emacs/gdb_stegotorus.el and run the specific line C-x C-e
+
+  in another one for client 
+
+  open ../gdb_emacs/gdb_stegotorus_c.el and run the specific line C-x C-e
+
+

doc/how_to_srcape_covers.txt

@@ -5,6 +5,10 @@ edit start_urls ad allowed_domains to reflect your cover providers
 
 then in the stegossidea's root type
 
-scrapy crawl stego_cover -o mititems.csv -t csv
+scrapy crawl stego_cover -o outputs.csv -t csv
 
 copy the file in stegotorus/apache_payload
+
+delete stegotorus/apache_payload/server_list.txt
+
+NOT REALLY? You need curlftps install for stegotorus to be able to scrap the covers for capacity computation

doc/writing_new_file_steg_mod_for_http_steg.txt

@@ -12,5 +12,5 @@ Functions.
 
 - in http.cc constructor calls the steg module constructor to make new steg modes.
 
-- Update [Apache]PayloadServer::find_url_type approperiately.
+- Update PayloadServer::extension_to_content_type
 

src/curl_util.cc

@@ -72,6 +72,8 @@ discard_data(char *ptr, size_t size, size_t nmemb, void *userdata)
     @param payload_length the length of the requested file this is equal to
     the size of allocated memory for the buf
     @param buf the alocated memory to store the POST reply
+
+    @return 0 if it fails to retrieve the url
 */
 unsigned long fetch_url_raw(CURL* curl_obj, string& url,  stringstream& buf)
 {
@@ -92,6 +94,7 @@ unsigned long fetch_url_raw(CURL* curl_obj, string& url,  stringstream& buf)
   }
 
   log_debug("read total bytes of : %lu:", buf.str().size());
+  log_assert(buf.str()[0]=='H');
   return buf.tellp();
 
 }

src/protocol/chop.cc

@@ -775,6 +775,13 @@ chop_circuit_t::send_all_steg_data()
       return 0;
   }
 
+  //It makes sense to check if there is any steg data to be sent
+  //otherwise we can just return.
+  //Now we check if the protocol_data has any data
+  size_t avail = evbuffer_get_length(target->steg->cfg()->protocol_data_out);
+  if (avail == 0)
+    return 0;
+
   bool no_target_connection = true;
 
   //TODO: Instead of re-implementing pick connection here I should 
@@ -800,8 +807,6 @@ chop_circuit_t::send_all_steg_data()
 
     size_t avail0;
 
-    //Now we check if the protocol_data has any data
-    size_t avail = evbuffer_get_length(target->steg->cfg()->protocol_data_out);
     //we try to send all data
     //we don't send random block or retransmit from the transmit
     //queue because we are called by the this->send which will
@@ -1822,6 +1827,7 @@ chop_conn_t::recv()
   if (config->mode == LSN_SIMPLE_SERVER && config->transparent_proxy) {
     received_length = evbuffer_get_length(bufferevent_get_input(buffer));
     originally_received = new uint8_t[received_length];
+    log_assert(originally_received);
     if (evbuffer_copyout(bufferevent_get_input(buffer), originally_received, received_length) != (ssize_t) received_length)
       log_abort("was not able to make a copy of received data");
   }

src/steg/apache_payload_server.cc

@@ -107,7 +107,6 @@ ApachePayloadServer::ApachePayloadServer(MachineSide init_side, const string& da
     }
 
   }
-
 
   //init curl
   if (!(_curl_obj = curl_easy_init()))
@@ -156,6 +155,7 @@ ApachePayloadServer::get_payload( int contentType, int cap, char** buf, int* siz
            cur_payload_candidate->length/(double)cap < noise2signal)) {
     itr_payloads++; numCandidate++;
     cur_payload_candidate = &_payload_database.payloads[itr_payloads->url_hash];
+
     }
 
     if (itr_payloads != _payload_database.sorted_payloads.end() && cur_payload_candidate->length < c_max_buffer_size)
@@ -196,7 +196,8 @@ ApachePayloadServer::get_payload( int contentType, int cap, char** buf, int* siz
                 numCandidate,
                 cap);
 
-      string best_payload = _payload_cache((itr_best->absolute_url_is_absolute ? "" : "http://" + _apache_host_name + "/") + (itr_best->absolute_url));
+      string& best_payload = _payload_cache((itr_best->absolute_url_is_absolute ? "" : "http://" + _apache_host_name + "/") + (itr_best->absolute_url)); //this is a permanent object in cache so it is ok to get a reference to it.
+      //if curl fails the size will be zero.
       *buf = (char*)best_payload.c_str();
       *size = best_payload.length();
       if (payload_id_hash)
@@ -225,8 +226,10 @@ ApachePayloadServer::fetch_hashed_url(const string& url)
 
   log_debug("asking cover server for payload %s", payload_uri.c_str());
   size_t payload_size = fetch_url_raw(_curl_obj, payload_uri, tmp_stream_buf);
-  if (payload_size) {
-      log_warn("Failed fetch the url %s", payload_uri.c_str());
+  if (payload_size == 0) {
+    log_warn("Failed fetch the url %s", payload_uri.c_str()); //here we should signal that we failed
+    //to retreieve the file and mark it as unacceptable
+    return string();
   }
 
   return tmp_stream_buf.str();
@@ -291,6 +294,8 @@ ApachePayloadServer::export_dict(iostream& dict_stream)
     {
       dict_stream << itr_uri->URL.c_str() << endl;
     }
+
+  log_debug("uri dictionary of size %ld has been exported.", uri_dict.size());
 
 }
 
@@ -352,32 +357,11 @@ ApachePayloadServer::find_url_type(const char* uri)
   log_debug("filename %s", filename.c_str());
   size_t last_dot = filename.rfind(".");
   if (last_dot == string::npos) 
-    ext = ".html"; //no filename assume html
+    ext = "html"; //no filename assume html
   else
-    ext = filename.substr(last_dot);
+    ext = filename.substr(last_dot+1);
 
   log_debug("ext %s", ext.c_str());
-  if (ext == ".html" || ext == ".htm" || ext == ".php"
-      || ext == ".jsp" || ext == ".asp")
-    return HTTP_CONTENT_HTML;
-
-  if (ext == ".js" || ext ==".JS")
-    return HTTP_CONTENT_JAVASCRIPT;
-
-  if ( ext ==".pdf" || ext == ".PDF")
-    return HTTP_CONTENT_PDF;
-
-  if (ext ==".swf" || ext == ".SWF")
-    return HTTP_CONTENT_SWF;
-
-  if (ext == ".png" || ext == ".PNG")
-    return HTTP_CONTENT_PNG;
-
-  if (ext == ".jpg" || ext == ".JPG")
-    return HTTP_CONTENT_JPEG;
-
-  if (ext == ".gif" || ext == ".GIF")
-    return HTTP_CONTENT_GIF;
-
-  return 0;
+  return extension_to_content_type(ext.c_str());
+
 }

src/steg/apache_payload_server.h

@@ -143,6 +143,10 @@ class ApachePayloadServer: public PayloadServer
    */
   virtual void disqualify_payload(const std::string& payload_id_hash) {
     _payload_database.payloads[payload_id_hash].corrupted = true;
+
+    //if the disqualified cover is the highest capacity cover then we need to
+    //decrease the max capacity
+    _payload_database.adjust_type_max_capacity(payload_id_hash);
   }
 
   /** 

src/steg/http.cc

@@ -34,6 +34,7 @@ using namespace std;
 #include "http_steg_mods/jpgSteg.h"
 #include "http_steg_mods/pngSteg.h"
 #include "http_steg_mods/gifSteg.h"
+#include "http_steg_mods/htmlSteg.h"
 
 #include "http.h"
 
@@ -125,9 +126,8 @@ void http_steg_config_t::init_file_steg_mods()
   file_steg_mods[HTTP_CONTENT_GIF] = new GIFSteg(payload_server, noise2signal);
   file_steg_mods[HTTP_CONTENT_SWF] = new SWFSteg(payload_server, noise2signal);
   file_steg_mods[HTTP_CONTENT_PDF] = new PDFSteg(payload_server, noise2signal);
-  file_steg_mods[HTTP_CONTENT_HTML] = new JSSteg(payload_server, noise2signal); //TODO: It should be a
-  //childo fo JS but for test only we are doing this
   file_steg_mods[HTTP_CONTENT_JAVASCRIPT] = new JSSteg(payload_server, noise2signal);
+  file_steg_mods[HTTP_CONTENT_HTML] = new HTMLSteg(payload_server, noise2signal);
 
 
   //TODO: for now only one steg module can be mentioned for testing.
@@ -148,9 +148,8 @@ void http_steg_config_t::init_file_steg_mods()
   if (http_steg_user_configs.find("steg_mod") != http_steg_user_configs.end()) {
     payload_server->set_active_steg_mods(http_steg_user_configs["steg_mod"]);
   }
-
-
 }
+
 http_steg_config_t::http_steg_config_t(config_t *cfg, const std::vector<std::string>& options)
   : steg_config_t(cfg),
     is_clientside(cfg->mode != LSN_SIMPLE_SERVER)

src/steg/http_apache.cc

@@ -169,6 +169,8 @@ http_apache_steg_config_t::http_apache_steg_config_t(config_t *cfg, const std::v
   string payload_filename;
   string cover_server, cover_list;
 
+  store_options(options);
+
   if (is_clientside)
     payload_filename = "apache_payload/client_list.txt";
   else {
@@ -181,8 +183,8 @@ http_apache_steg_config_t::http_apache_steg_config_t(config_t *cfg, const std::v
       cover_server = cfg->steg_mod_user_configs["protocol"]["cover_server"] != "" ?
         cfg->steg_mod_user_configs["protocol"]["cover_server"] : cover_server;
 
-      cover_list = cfg->steg_mod_user_configs["protocol"]["cover_list"] != "" ?
-        cfg->steg_mod_user_configs["protocol"]["cover_list"] : "";
+      cover_list = http_steg_user_configs["cover_list"] != "" ?
+        http_steg_user_configs["cover_list"] : "";
     }
 
   }