Merge pull request #606 from yahooguntu/master

Drastically speed up imports when bulk_size is specified
toptal · Mar 10, 2018 · f9c02dd · f9c02dd
2 parents eb1ad99 + d84b6cd
commit f9c02dd
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/lib/chewy/type/import/bulk_request.rb b/lib/chewy/type/import/bulk_request.rb
@@ -54,15 +54,17 @@ def request_base
 
         def request_bodies(body)
           if @bulk_size
+            serializer = ::Elasticsearch::API.serializer
             pieces = body.each_with_object(['']) do |piece, result|
               operation, meta = piece.to_a.first
               data = meta.delete(:data)
-              piece = [{operation => meta}, data].compact.map(&:to_json).join("\n")
+              piece = serializer.dump(operation => meta)
+              piece << "\n" << serializer.dump(data) if data.present?
 
               if result.last.bytesize + piece.bytesize > @bulk_size
                 result.push(piece)
               else
-                result[-1] = [result[-1], piece].reject(&:blank?).join("\n")
+                result[-1].blank? ? (result[-1] = piece) : (result[-1] << "\n" << piece)
               end
             end
             pieces.each { |piece| piece << "\n" }

diff --git a/lib/chewy/type/import/journal_builder.rb b/lib/chewy/type/import/journal_builder.rb
@@ -31,7 +31,7 @@ def entries(action, objects)
             index_name: @type.index.derivable_name,
             type_name: @type.type_name,
             action: action,
-            references: identify(objects).map(&:to_json).map(&Base64.method(:encode64)),
+            references: identify(objects).map { |item| Base64.encode64(::Elasticsearch::API.serializer.dump(item)) },
             created_at: Time.now.utc
           }
         end