From 8a9a6bbeb01054ef878640b595be7e7da2298f8c Mon Sep 17 00:00:00 2001 From: Sergey Kintsel Date: Fri, 27 May 2016 11:48:59 +0700 Subject: [PATCH] Add unlimited method to fetch all records from ES --- lib/chewy/query.rb | 23 ++++++++++++++++------- lib/chewy/query/criteria.rb | 14 ++++++++++++-- lib/chewy/search.rb | 2 +- spec/chewy/query_spec.rb | 18 ++++++++++++++++-- 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/lib/chewy/query.rb b/lib/chewy/query.rb index 2d3aeb757..7a82ea1f7 100644 --- a/lib/chewy/query.rb +++ b/lib/chewy/query.rb @@ -289,8 +289,8 @@ def timeout value # size: 100 # }} # - def limit value - chain { criteria.update_request_options size: Integer(value) } + def limit value = nil, &block + chain { criteria.update_request_options size: block || Integer(value) } end # Sets elasticsearch from search request param @@ -301,8 +301,8 @@ def limit value # from: 300 # }} # - def offset value - chain { criteria.update_request_options from: Integer(value) } + def offset value = nil, &block + chain { criteria.update_request_options from: block || Integer(value) } end # Elasticsearch highlight query option support @@ -894,8 +894,8 @@ def types! *params # scope = UsersIndex.aggs(max_age: { max: { field: 'age' } }).search_type(:count) # max_age = scope.aggs['max_age']['value'] # - def search_type val - chain { options.merge!(search_type: val) } + def search_type value + chain { criteria.update_search_options search_type: value } end # Merges two queries. @@ -961,6 +961,16 @@ def exists? search_type(:count).total > 0 end + # Sets limit to be equal to total documents count + # + # PlacesIndex.query(...).filter(...).unlimited + # + + def unlimited + count_query = search_type(:count) + offset(0).limit { count_query.total } + end + # Returns request total time elapsed as reported by elasticsearch # # UsersIndex.query(...).filter(...).took @@ -1004,7 +1014,6 @@ def _request @_request ||= begin request = criteria.request_body request.merge!(index: _indexes.map(&:index_name), type: _types.map(&:type_name)) - request.merge!(search_type: options[:search_type]) if options[:search_type] request end end diff --git a/lib/chewy/query/criteria.rb b/lib/chewy/query/criteria.rb index 1eaaf4f0c..be49e4c2c 100644 --- a/lib/chewy/query/criteria.rb +++ b/lib/chewy/query/criteria.rb @@ -5,7 +5,7 @@ class Query class Criteria include Compose ARRAY_STORAGES = [:queries, :filters, :post_filters, :sort, :fields, :types, :scores] - HASH_STORAGES = [:options, :request_options, :facets, :aggregations, :suggest, :script_fields] + HASH_STORAGES = [:options, :search_options, :request_options, :facets, :aggregations, :suggest, :script_fields] STORAGES = ARRAY_STORAGES + HASH_STORAGES def initialize options = {} @@ -48,6 +48,10 @@ def update_request_options(modifier) request_options.merge!(modifier) end + def update_search_options(modifier) + search_options.merge!(modifier) + end + def update_facets(modifier) facets.merge!(modifier) end @@ -120,7 +124,7 @@ def request_body body = _boost_query(body) - { body: body.merge!(request_options) } + { body: body.merge!(_request_options) }.merge!(search_options) end end @@ -154,6 +158,12 @@ def _boost_query(body) body.tap { |b| b[:query] = { function_score: score } } end + def _request_options + Hash[request_options.map do |key, value| + [key, value.is_a?(Proc) ? value.call : value] + end] + end + def _request_query _queries_join(queries, options[:query_mode]) end diff --git a/lib/chewy/search.rb b/lib/chewy/search.rb index 45099d5fa..262466009 100644 --- a/lib/chewy/search.rb +++ b/lib/chewy/search.rb @@ -10,7 +10,7 @@ module Search :boost_factor, :weight, :random_score, :field_value_factor, :decay, :aggregations, :suggest, :none, :strategy, :query, :filter, :post_filter, :boost_mode, :score_mode, :order, :reorder, :only, :types, :delete_all, :find, :total, - :total_count, :total_entries, to: :all + :total_count, :total_entries, :unlimited, to: :all end module ClassMethods diff --git a/spec/chewy/query_spec.rb b/spec/chewy/query_spec.rb index 89c459a7f..ac44f9996 100644 --- a/spec/chewy/query_spec.rb +++ b/spec/chewy/query_spec.rb @@ -96,6 +96,7 @@ specify { expect(subject.limit(10)).not_to eq(subject) } specify { expect(subject.limit(10).criteria.request_options).to include(size: 10) } specify { expect { subject.limit(10) }.not_to change { subject.criteria.request_options } } + specify { expect(subject.offset { 20/2 }.criteria.request_body[:body]).to include(size: 10) } end describe '#offset' do @@ -103,6 +104,7 @@ specify { expect(subject.offset(10)).not_to eq(subject) } specify { expect(subject.offset(10).criteria.request_options).to include(from: 10) } specify { expect { subject.offset(10) }.not_to change { subject.criteria.request_options } } + specify { expect(subject.offset { 20/2 }.criteria.request_body[:body]).to include(from: 10) } end describe '#script_fields' do @@ -454,7 +456,7 @@ end describe '#exists?' do - let(:data) { 10.times.map { |i| {id: i.next.to_s, name: "Name#{i.next}", age: 10 * i.next}.stringify_keys! } } + let(:data) { Array.new(10) { |i| {id: i.next.to_s, name: "Name#{i.next}", age: 10 * i.next}.stringify_keys! } } before { ProductsIndex::Product.import!(data.map { |h| double(h) }) } @@ -464,6 +466,18 @@ specify { expect(subject.filter(range: {age: {lt: 0}}).exists?).to eq false } end + describe '#unlimited' do + let(:data_length) { 10 } + let(:data) { Array.new(data_length) { |i| {id: i.next.to_s, name: "Name#{i.next}", age: 10 * i.next}.stringify_keys! } } + + before { ProductsIndex::Product.import!(data.map { |h| double(h) }) } + + specify { expect(subject.unlimited.count).to eq data_length } + specify { expect(subject.offset(5).unlimited.count).to eq data_length } + specify { expect(subject.limit(1).unlimited.count).to eq data_length } + specify { expect(subject.unlimited.limit(1).count).to eq 1 } + end + describe '#none' do specify { expect(subject.none).to be_a described_class } specify { expect(subject.none).not_to eq(subject) } @@ -577,7 +591,7 @@ end describe '#search_type' do - specify { expect(subject.search_type(:count).options).to include(search_type: :count) } + specify { expect(subject.search_type(:count).criteria.search_options).to include(search_type: :count) } end describe '#aggregations' do