Drastically speeds up imports when bulk_size is specified

toptal · Nov 9, 2017 · 614beaa · 614beaa
1 parent 1a13f75
commit 614beaa
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/lib/chewy/type/import/bulk_request.rb b/lib/chewy/type/import/bulk_request.rb
@@ -54,15 +54,17 @@ def request_base
 
         def request_bodies(body)
           if @bulk_size
+            serializer = ::Elasticsearch::API.serializer
             pieces = body.each_with_object(['']) do |piece, result|
               operation, meta = piece.to_a.first
               data = meta.delete(:data)
-              piece = [{operation => meta}, data].compact.map(&:to_json).join("\n")
+              piece = serializer.dump(operation => meta)
+              piece << "\n" << serializer.dump(data) if data.present?
 
               if result.last.bytesize + piece.bytesize > @bulk_size
                 result.push(piece)
               else
-                result[-1] = [result[-1], piece].reject(&:blank?).join("\n")
+                result[-1].blank? ? (result[-1] = piece) : (result[-1] << "\n" << piece)
               end
             end
             pieces.each { |piece| piece << "\n" }

diff --git a/lib/chewy/type/import/journal_builder.rb b/lib/chewy/type/import/journal_builder.rb
@@ -31,7 +31,7 @@ def entries(action, objects)
             index_name: @type.index.derivable_name,
             type_name: @type.type_name,
             action: action,
-            references: identify(objects).map(&:to_json),
+            references: identify(objects).map{ |item| ::Elasticsearch::API.serializer.dump(item) },
             created_at: Time.now.utc
           }
         end