Skip to content

Commit

Permalink
Add option to print progress while an index imports
Browse files Browse the repository at this point in the history
  • Loading branch information
Graham Kaemmer committed Jan 18, 2017
1 parent eb787f9 commit 5d77276
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 3 deletions.
4 changes: 2 additions & 2 deletions lib/chewy/index/actions.rb
Original file line number Diff line number Diff line change
Expand Up @@ -160,10 +160,10 @@ def #{method} options = {}
#
# UsersIndex.reset! Time.now.to_i, journal: true
#
def reset!(suffix = nil, journal: false)
def reset!(suffix = nil, journal: false, progress: false)
if suffix.present? && (indexes = self.indexes).present?
create! suffix, alias: false
result = import suffix: suffix, journal: journal
result = import suffix: suffix, journal: journal, progress: progress
client.indices.update_aliases body: { actions: [
*indexes.map do |index|
{ remove: { index: index, alias: index_name } }
Expand Down
2 changes: 1 addition & 1 deletion lib/chewy/rake_helper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def reset_index(*indexes)
normalize_indexes(indexes).each do |index|
puts "Resetting #{index}"
time = Time.now
index.reset!((time.to_f * 1000).round)
index.reset!((time.to_f * 1000).round, progress: ENV['PROGRESS'])
if index.journal?
Chewy::Journal.create
Chewy::Journal::Apply.since(time, only: [index])
Expand Down
6 changes: 6 additions & 0 deletions lib/chewy/type/import.rb
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,15 @@ module ClassMethods
# UsersIndex::User.import bulk_size: 10.megabytes # import ElasticSearch bulk size in bytes
# UsersIndex::User.import consistency: :quorum # explicit write consistency setting for the operation (one, quorum, all)
# UsersIndex::User.import replication: :async # explicitly set the replication type (sync, async)
# UsersIndex::User.import progress: true # print progress as import takes place (after each batch)
#
# See adapters documentation for more details.
#
def import(*args)
import_options = args.extract_options!
import_options.reverse_merge! _default_import_options
bulk_options = import_options.reject { |k, _| !BULK_OPTIONS.include?(k) }.reverse_merge!(refresh: true)
total_imported = 0

index.create!(bulk_options.slice(:suffix)) unless index.exists?

Expand All @@ -42,6 +44,10 @@ def import(*args)

fill_payload_import payload, action_objects
fill_payload_errors payload, errors if errors.present?

total_imported += action_objects.values.flatten.length
print "Imported #{total_imported} items\r" if import_options[:progress]

!errors.present?
end
end
Expand Down
21 changes: 21 additions & 0 deletions spec/chewy/type/import_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,27 @@
city.import dummy_cities, refresh: false
end

context ':progress' do
specify do
expect do
city.import dummy_cities, progress: true
end.to output("Imported 3 items\r").to_stdout
end

specify do
expect do
city.import dummy_cities, progress: true, batch_size: 2
end.to output("Imported 2 items\rImported 3 items\r").to_stdout
end

specify do
expect do
city.import dummy_cities, progress: true, batch_size: 1
end.to output("Imported 1 items\rImported 2 items\rImported 3 items\r").to_stdout
end
end


context 'scoped' do
before do
names = %w(name0 name1)
Expand Down

0 comments on commit 5d77276

Please sign in to comment.