Skip to content

Commit

Permalink
Add support for namespaced indexes in journaling (#432)
Browse files Browse the repository at this point in the history 
* Add support for namespaced indexes in journaling
  • Loading branch information
@pyromaniac
François-Pierre Bouchard authored and pyromaniac committed Sep 27, 2016
1 parent f087b9f commit 1666e99
Show file tree
Hide file tree
Showing 5 changed files with 168 additions and 156 deletions.
4 changes: 4 additions & 0 deletions lib/chewy/index.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ def self._index_name(suggest = nil)
@_index_name
end

def self.derivable_index_name
@_derivable_index_name ||= name.sub(/Index\Z/, '').underscore
end

# Setups or returns pure Elasticsearch index name
# without any prefixes/suffixes
def self.default_prefix
Expand Down
2 changes: 1 addition & 1 deletion lib/chewy/journal.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def add(action_objects)
@records +=
action_objects.map do |action, objects|
{
index_name: @index._index_name,
index_name: @index.derivable_index_name,
type_name: @index.type_name,
action: action,
object_ids: identify(objects),
Expand Down
2 changes: 1 addition & 1 deletion lib/chewy/type.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ class Type
include Witchcraft
include Import

singleton_class.delegate :index_name, :_index_name, :client, to: :index
singleton_class.delegate :index_name, :_index_name, :derivable_index_name, :client, to: :index

class_attribute :_default_import_options
self._default_import_options = {}
Expand Down
314 changes: 160 additions & 154 deletions spec/chewy/journal_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,165 +2,171 @@

describe Chewy::Journal do
context 'journaling', orm: true do
before do
stub_model(:city) do
update_index 'places#city', :self
end
stub_model(:country) do
update_index 'places#country', :self
end

stub_index(:places) do
define_type City do
default_import_options journal: true
end
define_type Country do
default_import_options journal: true
['', 'namespace/'].each do |namespace|
context namespace.present? ? 'with namespace' : 'without namespace' do
before do
stub_model(:city) do
update_index "#{namespace}places#city", :self
end
stub_model(:country) do
update_index "#{namespace}places#country", :self
end

stub_index("#{namespace}places") do
define_type City do
default_import_options journal: true
end
define_type Country do
default_import_options journal: true
end
end

Chewy.massacre
begin
Chewy.client.indices.delete(index: Chewy::Journal.index_name)
rescue Elasticsearch::Transport::Transport::Errors::NotFound
nil
end
Chewy.settings[:prefix] = 'some_prefix'
Timecop.freeze(time)
end
end

Chewy.massacre
begin
Chewy.client.indices.delete(index: Chewy::Journal.index_name)
rescue Elasticsearch::Transport::Transport::Errors::NotFound
nil
end
Chewy.settings[:prefix] = 'some_prefix'
Timecop.freeze(time)
end

after do
Chewy.settings[:prefix] = nil
Timecop.return
end
after do
Chewy.settings[:prefix] = nil
Timecop.return
end

let(:time) { Time.now }
let(:import_time) { time + 1 }
let(:update_time) { time + 2 }
let(:destroy_time) { time + 3 }
let(:time) { Time.now }
let(:import_time) { time + 1 }
let(:update_time) { time + 2 }
let(:destroy_time) { time + 3 }

def timestamp(time)
time.to_i
end
def timestamp(time)
time.to_i
end

specify do
Chewy.strategy(:urgent) do
cities = Array.new(2) { |i| City.create!(id: i + 1) }
countries = Array.new(2) { |i| Country.create!(id: i + 1) }
Country.create!(id: 3)

Timecop.freeze(import_time)
PlacesIndex.import

expect(Chewy.client.indices.exists?(index: Chewy::Journal.index_name)).to eq true

Timecop.freeze(update_time)
cities.first.update_attributes!(name: 'Supername')

Timecop.freeze(destroy_time)
countries.last.destroy

journal_records = Chewy.client.search(index: Chewy::Journal.index_name, type: Chewy::Journal.type_name, sort: 'created_at')['hits']['hits'].map { |r| r['_source'] }

expected_journal = [
{
'index_name' => 'places',
'type_name' => 'city',
'action' => 'index',
'object_ids' => [1],
'created_at' => time.to_i
},
{
'index_name' => 'places',
'type_name' => 'city',
'action' => 'index',
'object_ids' => [2],
'created_at' => time.to_i
},
{
'index_name' => 'places',
'type_name' => 'country',
'action' => 'index',
'object_ids' => [1],
'created_at' => time.to_i
},
{
'index_name' => 'places',
'type_name' => 'country',
'action' => 'index',
'object_ids' => [2],
'created_at' => time.to_i
},
{
'index_name' => 'places',
'type_name' => 'country',
'action' => 'index',
'object_ids' => [3],
'created_at' => time.to_i
},
{
'index_name' => 'places',
'type_name' => 'city',
'action' => 'index',
'object_ids' => [1, 2],
'created_at' => import_time.to_i
},
{
'index_name' => 'places',
'type_name' => 'country',
'action' => 'index',
'object_ids' => [1, 2, 3],
'created_at' => import_time.to_i
},
{
'index_name' => 'places',
'type_name' => 'city',
'action' => 'index',
'object_ids' => [1],
'created_at' => update_time.to_i
},
{
'index_name' => 'places',
'type_name' => 'country',
'action' => 'delete',
'object_ids' => [2],
'created_at' => destroy_time.to_i
}
]

expect(Chewy.client.count(index: Chewy::Journal.index_name)['count']).to eq 9
expect(journal_records).to eq expected_journal

journal_entries = Chewy::Journal.entries_from(import_time)
expect(journal_entries.length).to eq 4
# we have only 2 types, so we can group all journal entries(4) into 2
expect(Chewy::Journal.group(journal_entries)).to eq [
Chewy::Journal::Entry.new('index_name' => 'places',
'type_name' => 'city',
'action' => nil,
'object_ids' => [1, 2],
'created_at' => nil),
Chewy::Journal::Entry.new('index_name' => 'places',
'type_name' => 'country',
'action' => 'delete',
'object_ids' => [1, 2, 3],
'created_at' => destroy_time.to_i)
]

# simulate lost data
Chewy.client.delete(index: "#{Chewy.settings[:prefix]}_places", type: 'city', id: 1, refresh: true)
expect(PlacesIndex::City.all.to_a.length).to eq 1

Chewy::Journal.apply_changes_from(time)
expect(PlacesIndex::City.all.to_a.length).to eq 2

expect(Chewy::Journal.clean_until(import_time)).to eq 7
expect(Chewy.client.count(index: Chewy::Journal.index_name)['count']).to eq 2

expect(Chewy::Journal.delete!).to be_truthy
expect { Chewy::Journal.delete! }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
expect(Chewy::Journal.delete).to eq false
expect(Chewy::Journal.exists?).to eq false
specify do
places_index = namespace.present? ? Namespace::PlacesIndex : PlacesIndex
Chewy.strategy(:urgent) do
cities = Array.new(2) { |i| City.create!(id: i + 1) }
countries = Array.new(2) { |i| Country.create!(id: i + 1) }
Country.create!(id: 3)

Timecop.freeze(import_time)

places_index.import

expect(Chewy.client.indices.exists?(index: Chewy::Journal.index_name)).to eq true

Timecop.freeze(update_time)
cities.first.update_attributes!(name: 'Supername')

Timecop.freeze(destroy_time)
countries.last.destroy

journal_records = Chewy.client.search(index: Chewy::Journal.index_name, type: Chewy::Journal.type_name, sort: 'created_at')['hits']['hits'].map { |r| r['_source'] }

expected_journal = [
{
'index_name' => "#{namespace}places",
'type_name' => 'city',
'action' => 'index',
'object_ids' => [1],
'created_at' => time.to_i
},
{
'index_name' => "#{namespace}places",
'type_name' => 'city',
'action' => 'index',
'object_ids' => [2],
'created_at' => time.to_i
},
{
'index_name' => "#{namespace}places",
'type_name' => 'country',
'action' => 'index',
'object_ids' => [1],
'created_at' => time.to_i
},
{
'index_name' => "#{namespace}places",
'type_name' => 'country',
'action' => 'index',
'object_ids' => [2],
'created_at' => time.to_i
},
{
'index_name' => "#{namespace}places",
'type_name' => 'country',
'action' => 'index',
'object_ids' => [3],
'created_at' => time.to_i
},
{
'index_name' => "#{namespace}places",
'type_name' => 'city',
'action' => 'index',
'object_ids' => [1, 2],
'created_at' => import_time.to_i
},
{
'index_name' => "#{namespace}places",
'type_name' => 'country',
'action' => 'index',
'object_ids' => [1, 2, 3],
'created_at' => import_time.to_i
},
{
'index_name' => "#{namespace}places",
'type_name' => 'city',
'action' => 'index',
'object_ids' => [1],
'created_at' => update_time.to_i
},
{
'index_name' => "#{namespace}places",
'type_name' => 'country',
'action' => 'delete',
'object_ids' => [2],
'created_at' => destroy_time.to_i
}
]

expect(Chewy.client.count(index: Chewy::Journal.index_name)['count']).to eq 9
expect(journal_records).to eq expected_journal

journal_entries = Chewy::Journal.entries_from(import_time)
expect(journal_entries.length).to eq 4
# we have only 2 types, so we can group all journal entries(4) into 2
expect(Chewy::Journal.group(journal_entries)).to eq [
Chewy::Journal::Entry.new('index_name' => "#{namespace}places",
'type_name' => 'city',
'action' => nil,
'object_ids' => [1, 2],
'created_at' => nil),
Chewy::Journal::Entry.new('index_name' => "#{namespace}places",
'type_name' => 'country',
'action' => 'delete',
'object_ids' => [1, 2, 3],
'created_at' => destroy_time.to_i)
]

# simulate lost data
Chewy.client.delete(index: "#{Chewy.settings[:prefix]}_places", type: 'city', id: 1, refresh: true)
expect(places_index::City.all.to_a.length).to eq 1

Chewy::Journal.apply_changes_from(time)
expect(places_index::City.all.to_a.length).to eq 2

expect(Chewy::Journal.clean_until(import_time)).to eq 7
expect(Chewy.client.count(index: Chewy::Journal.index_name)['count']).to eq 2

expect(Chewy::Journal.delete!).to be_truthy
expect { Chewy::Journal.delete! }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
expect(Chewy::Journal.delete).to eq false
expect(Chewy::Journal.exists?).to eq false
end
end
end
end
end
Expand Down
2 changes: 2 additions & 0 deletions spec/chewy_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
it { is_expected.to be < Chewy::Type }
its(:name) { should == 'CitiesIndex::City' }
its(:index) { should == CitiesIndex }
its(:derivable_index_name) { should == 'cities' }
its(:type_name) { should == 'city' }
end

Expand All @@ -77,6 +78,7 @@
it { is_expected.to be < Chewy::Type }
its(:name) { should == 'Namespace::CitiesIndex::City' }
its(:index) { should == Namespace::CitiesIndex }
its(:derivable_index_name) { should == 'namespace/cities' }
its(:type_name) { should == 'city' }
end

Expand Down

0 comments on commit 1666e99

Please sign in to comment.