Bugfix for terrible, terrible BSON Binary object serialization/desera…

…lization bug

.gitignore

@@ -4,4 +4,5 @@
 .project
 .settings
 data
-node_modules
+node_modules
+output

deps/nodeunit/lib/reporters/junit.js

@@ -150,9 +150,11 @@ exports.run = function (files, opts, callback) {
                         var rendered = ejs.render(tmpl, {
                             locals: {suites: [module]}
                         });
+                        var suffix = opts.suffix == null ? '' : "_" + opts.suffix;
+
                         var filename = path.join(
                             opts.output,
-                            module.name + '.xml'
+                            module.name + suffix + '.xml'
                         );
                         sys.puts('Writing ' + filename);
                         fs.writeFile(filename, rendered, cb);
@@ -173,8 +175,14 @@ exports.run = function (files, opts, callback) {
                                 ' assertions (' + assertions.duration + 'ms)'
                             );
                         }
-                        setTimeout(function () {
+                        var timer = setTimeout(function () {
+                           clearTimeout(timer);
+
+                          if(callback != null) {
+                            return callback();
+                          } else {                
                             process.reallyExit(assertions.failures());
+                          }              
                         }, 10);
                     });
 

external-libs/bson/bson.cc

@@ -349,18 +349,18 @@ uint32_t BSON::serialize(char *serialized_object, uint32_t index, Handle<Value>
     // Unpack the object and encode
     Local<Object> obj = value->ToObject();
     Binary *binary_obj = Binary::Unwrap<Binary>(obj);
-    // Let's write the total size of the binary 
-    BSON::write_int32((serialized_object + index), (binary_obj->index + 4));
+    // // Let's write the total size of the binary 
+    // BSON::write_int32((serialized_object + index), (binary_obj->index + 4));
+    // // Adjust index
+    // index = index + 4;
+    // Let's write the content to the char* array
+    BSON::write_int32((serialized_object + index), binary_obj->index);
     // Adjust index
     index = index + 4;
     // Write subtype
     *(serialized_object + index)  = (char)binary_obj->sub_type;
     // Adjust index
     index = index + 1;
-    // Let's write the content to the char* array
-    BSON::write_int32((serialized_object + index), binary_obj->index);
-    // Adjust index
-    index = index + 4;
     // Write binary content
     memcpy((serialized_object + index), binary_obj->data, binary_obj->index);
     // Adjust index
@@ -742,7 +742,7 @@ uint32_t BSON::calculate_object_size(Handle<Value> value) {
     Local<Object> obj = value->ToObject();
     Binary *binary_obj = Binary::Unwrap<Binary>(obj);
     // Adjust the object_size, binary content lengt + total size int32 + binary size int32 + subtype
-    object_size += binary_obj->index + 4 + 4 + 1;
+    object_size += binary_obj->index + 4 + 1;
   } else if(Code::HasInstance(value)) {
     // printf("================================ calculate_object_size:code\n");
     // Unpack the object and encode

lib/mongodb/bson/binary.js

@@ -4,7 +4,9 @@
  */
 
 var Buffer = require('buffer').Buffer; // TODO just use global Buffer
-var bson = require('./bson');
+var bson = require('./bson'),
+  debug = require('util').debug,
+  inspect = require('util').inspect;
 
 /**
  * Binary constructor.

lib/mongodb/bson/binary_parser.js

@@ -275,51 +275,54 @@ BinaryParser.hprint = function hprint (s) {
 
   for (var i = 0, len = s.length; i < len; i++) {
     if (s.charCodeAt(i) < 32) {
-
       number = s.charCodeAt(i) <= 15
         ? "0" + s.charCodeAt(i).toString(16)
-        : s.charCodeAt(i).toString(16);
-
+        : s.charCodeAt(i).toString(16);        
       process.stdout.write(number)
-
-      // sys.debug(number+' : ');
-
     } else {
-
       number = s.charCodeAt(i) <= 15
         ? "0" + s.charCodeAt(i).toString(16)
         : s.charCodeAt(i).toString(16);
-
         process.stdout.write(number)
-      // sys.debug(number+' : '+ s.charAt(i));
-
     }
   }
 
   process.stdout.write("\n\n");
 };
 
+BinaryParser.ilprint = function hprint (s) {
+  var number;
+
+  for (var i = 0, len = s.length; i < len; i++) {
+    if (s.charCodeAt(i) < 32) {
+      number = s.charCodeAt(i) <= 15
+        ? "0" + s.charCodeAt(i).toString(10)
+        : s.charCodeAt(i).toString(10);
+
+      sys.debug(number+' : ');
+    } else {
+      number = s.charCodeAt(i) <= 15
+        ? "0" + s.charCodeAt(i).toString(10)
+        : s.charCodeAt(i).toString(10);
+      sys.debug(number+' : '+ s.charAt(i));
+    }
+  }
+};
 
 BinaryParser.hlprint = function hprint (s) {
   var number;
 
   for (var i = 0, len = s.length; i < len; i++) {
     if (s.charCodeAt(i) < 32) {
-
       number = s.charCodeAt(i) <= 15
         ? "0" + s.charCodeAt(i).toString(16)
         : s.charCodeAt(i).toString(16);
-
       sys.debug(number+' : ');
-
     } else {
-
       number = s.charCodeAt(i) <= 15
         ? "0" + s.charCodeAt(i).toString(16)
         : s.charCodeAt(i).toString(16);
-
       sys.debug(number+' : '+ s.charAt(i));
-
     }
   }
 };

lib/mongodb/bson/bson.js

@@ -52,6 +52,10 @@ BSON.BSON_BINARY_SUBTYPE_USER_DEFINED = 128;
  */
 
 BSON.serialize = function serialize (data, checkKeys) {
+  // debug("======================================================================== START")
+  // BinaryParser.hprint(BSON.encodeValue('', null, data, true, checkKeys == null ? false : checkKeys))
+  // debug("======================================================================== END")
+
   return BSON.encodeValue('', null, data, true, checkKeys == null ? false : checkKeys);
 };
 
@@ -468,14 +472,13 @@ BSON.deserialize = function deserialize (data, is_array_item, returnData, return
         var string_name = data.substring(index, string_end_index);
 
         // UTF-8 decode key name
-        // string_name = decodeURIComponent(escape(string_name));
         string_name = BinaryParser.decode_utf8(string_name);
 
         // Ajust index to point to the end of the string
         index = string_end_index + 1;
 
-        // The total number of bytes after subtype
-        var total_number_of_bytes = BinaryParser.toInt(data.substr(index, 4));
+        // Read the size of the binary
+        var number_of_bytes = BinaryParser.toInt(data.substr(index, 4));
         index = index + 4;
 
         // Decode the subtype
@@ -486,10 +489,6 @@ BSON.deserialize = function deserialize (data, is_array_item, returnData, return
         var value = new Binary();
         value.sub_type = sub_type;
 
-        // Read the size of the binary
-        var number_of_bytes = BinaryParser.toInt(data.substr(index, 4));
-        index = index + 4;
-
         // Read the next bytes into our Binary object
         var bin_data = data.substr(index, number_of_bytes);
         value.write(bin_data);
@@ -714,8 +713,7 @@ BSON.encodeRegExp = function encodeRegExp (regexp) {
 
 BSON.encodeBinary = function encodeBinary (binary) {
   var data = binary.value();
-  return BinaryParser.fromByte(binary.sub_type)
-       + BinaryParser.fromInt(data.length) + data;
+  return BinaryParser.fromInt(data.length) + BinaryParser.fromByte(binary.sub_type) + data;
 };
 
 /**
@@ -860,12 +858,10 @@ BSON.encodeValue = function encodeValue (encoded_string, variable, value, top_le
     encoded_string += BinaryParser.fromByte(BSON.BSON_DATA_CODE_W_SCOPE)
                     + variable_encoded
                     + BSON.encodeCode(value, checkKeys);
-  } else if(value instanceof Binary || toString.call(value) == "[object Binary]") {
-    var object_string = BSON.encodeBinary(value);
+  } else if(value instanceof Binary || toString.call(value) == "[object Binary]") {    
     encoded_string += BinaryParser.fromByte(BSON.BSON_DATA_BINARY)
                     + variable_encoded
-                    + BSON.encodeInt(object_string.length - 1)
-                    + object_string;
+                    + BSON.encodeBinary(value);
   } else if(value instanceof DBRef) {
     var object_string = BSON.encodeDBRef(value);
     encoded_string += BinaryParser.fromByte(BSON.BSON_DATA_OBJECT)

lib/mongodb/commands/insert_command.js

@@ -1,6 +1,8 @@
 var BaseCommand = require('./base_command').BaseCommand,
   BinaryParser = require('../bson/binary_parser').BinaryParser,
-  inherits = require('util').inherits;
+  inherits = require('util').inherits,
+  debug = require('util').debug,
+  inspect = require('util').inspect;
 
 /**
   Insert Document Command
@@ -22,6 +24,7 @@ InsertCommand.prototype.add = function(document) {
 };
 
 InsertCommand.prototype.getOpCode = function() {
+
   return BaseCommand.OP_INSERT;
 };
 
@@ -35,7 +38,7 @@ struct {
 */
 InsertCommand.prototype.getCommand = function() {
   var command_string = '';
-  for(var i = 0; i < this.documents.length; i++) {
+  for(var i = 0; i < this.documents.length; i++) {    
     command_string = command_string + this.db.bson_serializer.BSON.serialize(this.documents[i], this.checkKeys);
   }
   // Build the command string

lib/mongodb/connection.js

@@ -112,13 +112,13 @@ Connection.prototype.send = function(command) {
     if ( this.connection.readyState != "open" )
       throw 'notConnected';
     if(command.constructor == String) {
-      // debug("========================================================================= command")
-      // BinaryParser.hprint(command)
-      
+      // debug("========================================================================= command string")
+      // BinaryParser.ilprint(command)      
+
       this.connection.write(command, "binary");      
     } else {
-      // debug("========================================================================= command")
-      // BinaryParser.hprint(command.toBinary())
+      // debug("========================================================================= command command")
+      // BinaryParser.ilprint(command.toBinary())
 
       this.connection.write(command.toBinary(), "binary");      
     }    
@@ -151,8 +151,12 @@ Connection.prototype.send = function(command) {
           while(self.messages.length > 0) {
             var msg = self.messages.shift();
             if(msg.constructor == String) {
+              // debug("========================================================================= write message")
+              // BinaryParser.hlprint(msg)              
               this.write(msg, "binary");      
             } else {
+              // debug("========================================================================= write message")
+              // BinaryParser.hlprint(msg.toBinary())
               this.write(msg.toBinary(), "binary");      
             }    
             // this.write(self.messages.shift().toBinary(), "binary");
@@ -176,6 +180,9 @@ Connection.prototype.sendwithoutReconnect = function(command) {
     throw new Error( 'Connection closed!' );
   }
   try {
+    // debug("========================================================================= write message")
+    // BinaryParser.hlprint(command.toBinary())
+
     this.connection.write(command.toBinary(), "binary");
   } catch(err) {
     // no need to reconnect since called by latest master

lib/mongodb/db.js

@@ -508,12 +508,10 @@ Db.prototype.executeCommand = function(db_command, read, callback) {
     };
 
     try{
-      // debug("========================================================= 3");
       var writer = read ? self.serverConfig.checkoutReader() : self.serverConfig.checkoutWriter();
       // debug(inspect(writer))
 
       writer.send(db_command)
-
       // self.serverConfig.primary.send(db_command);   
     } catch(err){
       if(callback instanceof Function) {        

lib/mongodb/gridfs/chunk.js

@@ -1,5 +1,7 @@
 var BinaryParser = require('../bson/binary_parser').BinaryParser,
-  sys = require('util');
+  sys = require('util'),
+  debug = require('util').debug,
+  inspect = require('util').inspect;
 
 /**
  * Class for representing a signle chunk in GridFS.
@@ -58,7 +60,8 @@ var Chunk = exports.Chunk = function(file, mongoObject) {
  *     will contain a reference to this object.
  */
 Chunk.prototype.write = function(data, callback) {
-  this.data.write(data.toString('binary'), this.internalPosition);
+  // this.data.write(data.toString('binary'), this.internalPosition);
+  this.data.write(data, this.internalPosition);
   this.internalPosition = this.data.length();
   callback(null, this);
 };
@@ -143,7 +146,8 @@ Chunk.prototype.save = function(callback) {
     collection.remove({'_id':self.objectId}, function(err, result) {
       if(self.data.length() > 0) {
         self.buildMongoObject(function(mongoObject) {
-          collection.insert(mongoObject, function(collection) {
+
+          collection.insert(mongoObject, function(err, collection) {
             callback(null, self);
           });
         });