Merge pull request #568 from coopdevs/remove_elastic_2

Remove Elastic, 2nd attemp

Author: sauloperez

.travis.yml

@@ -3,7 +3,6 @@ dist: bionic
 cache: bundler
 services:
   - postgresql
-  - elasticsearch
 addons:
   postgresql: "9.4"
   chrome: stable
@@ -19,7 +18,5 @@ before_script:
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter
   - ./cc-test-reporter before-build
-  # allow elasticsearch to be ready - https://docs.travis-ci.com/user/database-setup/#ElasticSearch
-  - sleep 10
 after_script:
   - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT

Gemfile

@@ -16,10 +16,9 @@ gem 'unicorn'
 gem 'kaminari', '~> 1.1.1'
 gem "simple_form", ">= 3.0.0"
 gem 'rollbar', '2.8.3'
+gem 'pg_search', '2.1.4'
 gem 'prawn', '~> 2.2.0'
 gem 'prawn-table', '~> 0.2.2'
-gem 'elasticsearch-model'
-gem 'elasticsearch-rails'
 gem 'skylight'
 gem 'sidekiq', '5.1.3'
 gem 'sidekiq-cron', '~> 1.1.0'

Gemfile.lock

@@ -118,19 +118,6 @@ GEM
     dotenv-rails (2.7.1)
       dotenv (= 2.7.1)
       railties (>= 3.2, < 6.1)
-    elasticsearch (1.0.8)
-      elasticsearch-api (= 1.0.7)
-      elasticsearch-transport (= 1.0.7)
-    elasticsearch-api (1.0.7)
-      multi_json
-    elasticsearch-model (0.1.7)
-      activesupport (> 3)
-      elasticsearch (> 0.4)
-      hashie
-    elasticsearch-rails (0.1.7)
-    elasticsearch-transport (1.0.7)
-      faraday
-      multi_json
     erubi (1.9.0)
     erubis (2.7.0)
     et-orbi (1.1.7)
@@ -139,8 +126,6 @@ GEM
     fabrication (2.20.1)
     faker (1.9.6)
       i18n (>= 0.7)
-    faraday (0.9.1)
-      multipart-post (>= 1.2, < 3)
     ffi (1.12.2)
     formtastic (3.1.5)
       actionpack (>= 3.2.13)
@@ -154,7 +139,6 @@ GEM
     has_scope (0.6.0)
       actionpack (>= 3.2, < 5)
       activesupport (>= 3.2, < 5)
-    hashie (3.4.1)
     hstore_translate (1.0.0)
       activerecord (>= 3.1.0)
     http-cookie (1.0.3)
@@ -207,7 +191,6 @@ GEM
     mini_portile2 (2.4.0)
     minitest (5.14.0)
     multi_json (1.11.2)
-    multipart-post (2.0.0)
     net-scp (2.0.0)
       net-ssh (>= 2.6.5, < 6.0.0)
     net-ssh (5.2.0)
@@ -220,6 +203,9 @@ GEM
       ast (~> 2.4.0)
     pdf-core (0.7.0)
     pg (0.21.0)
+    pg_search (2.1.4)
+      activerecord (>= 4.2)
+      activesupport (>= 4.2)
     prawn (2.2.2)
       pdf-core (~> 0.7.0)
       ttfunk (~> 1.5)
@@ -411,8 +397,6 @@ DEPENDENCIES
   database_cleaner (= 1.6.2)
   devise (~> 4.7.1)
   dotenv-rails (~> 2.7.1)
-  elasticsearch-model
-  elasticsearch-rails
   fabrication (~> 2.20)
   faker (~> 1.9)
   has_scope
@@ -423,6 +407,7 @@ DEPENDENCIES
   letter_opener (= 1.4.1)
   localeapp (= 2.1.1)
   pg (= 0.21.0)
+  pg_search (= 2.1.4)
   prawn (~> 2.2.0)
   prawn-table (~> 0.2.2)
   pundit (~> 2.0.0)

app/controllers/members_controller.rb

@@ -20,11 +20,6 @@ def toggle_manager
   def toggle_active
     find_member
     @member.toggle(:active).save!
-    if @member.active
-      @member.add_all_posts_to_index
-    else
-      @member.remove_all_posts_from_index
-    end
     respond_to do |format|
       format.json { head :ok }
       format.html { redirect_to :back }

app/controllers/posts_controller.rb

@@ -4,31 +4,22 @@ class PostsController <  ApplicationController
   has_scope :by_organization, as: :org
 
   def index
-    if (query = params[:q]).present?
-      # match query term on fields
-      must = [ { multi_match: {
-          query: query.to_s,
-          type: "phrase_prefix",
-          fields: ["title^2", "description", "tags^2"]
-        } } ]
-      if current_organization.present?
-        # filter by organization
-        must << { term: { organization_id: { value: current_organization.id } } }
-      end
-      posts = model.__elasticsearch__.search(
-        query: {
-          bool: {
-            must: must
-          }
-        }
-      ).page(params[:page]).per(25).records
-    else
-      posts = model.active.of_active_members
-      if current_organization.present?
-        posts = posts.merge(current_organization.posts)
-      end
-      posts = apply_scopes(posts).page(params[:page]).per(25)
+    context = model.active.of_active_members
+    if current_organization.present?
+      context = context.where(
+        organization_id: current_organization.id
+      )
     end
+
+    posts = if (query = params[:q]).present?
+              context.
+                search_by_query(query).
+                page(params[:page]).
+                per(25)
+            else
+              apply_scopes(context).page(params[:page]).per(25)
+            end
+
     instance_variable_set("@#{resources}", posts)
   end
 

app/models/member.rb

@@ -44,18 +44,6 @@ def display_id
     member_uid
   end
 
-  def remove_all_posts_from_index
-    Post.with_member.where("members.id = ?", self.id).find_each do |post|
-      post.delete_document
-    end
-  end
-
-  def add_all_posts_to_index
-    Post.with_member.where("members.id = ?", self.id).find_each do |post|
-      post.update_or_delete_document(self)
-    end
-  end
-
   def assign_registration_number
     self.member_uid ||= organization.next_reg_number_seq
   end

app/models/post.rb

@@ -1,40 +1,16 @@
-require 'elasticsearch/model'
-
 class Post < ActiveRecord::Base
   include Taggable
-
-  # Elasticsearch::Model doesn't work well with STI, so
-  # include it in subclasses directly.
-  def self.inherited(child)
-    super
-
-    child.instance_eval do
-      include Elasticsearch::Model
-
-      after_commit :index_document, on: :create
-      after_commit :update_or_delete_document, on: :update
-      after_commit :delete_document, on: :destroy
-
-      settings(
-        analysis: {
-          analyzer: {
-            normal: {
-              tokenizer: "standard",
-              # lowercase, unaccent
-              filter: %w[lowercase asciifolding]
-            }
-          }
-        }
-      ) do
-        mapping do
-          indexes :title, analyzer: "normal"
-          indexes :description, analyzer: "normal"
-          indexes :tags
-          indexes :organization_id, type: :integer
-        end
-      end
-    end
-  end
+  include PgSearch
+
+  pg_search_scope :search_by_query,
+    against: [:title, :description, :tags],
+    ignoring: :accents,
+    using: {
+      tsearch: {
+        prefix: true,
+        tsvector_column: 'tsv'
+      }
+    }
 
   attr_reader :member_id
 
@@ -72,30 +48,6 @@ def self.inherited(child)
   validates :category, presence: true
   validates :title, presence: true
 
-  def index_document
-    __elasticsearch__.index_document
-  end
-
-  # pass member when doing bulk things
-  def update_or_delete_document(member = nil)
-    member ||= self.member
-    if active && member.try(:active)
-      begin
-        __elasticsearch__.update_document
-      rescue # document was not in the index. TODO: more specifi exception class
-        __elasticsearch__.index_document
-      end
-    else
-      __elasticsearch__.delete_document
-    end
-  rescue # document was not in the index. TODO: more specifi exception class
-  end
-
-  def delete_document
-    __elasticsearch__.delete_document
-  rescue # document was not in the index. TODO: more specifi exception class
-  end
-
   def as_indexed_json(*)
     as_json(only: [:title, :description, :tags, :organization_id])
   end

config/application.rb

@@ -35,5 +35,9 @@ class Application < Rails::Application
 
     # ActiveJob configuration
     config.active_job.queue_adapter = :sidekiq
+
+    # Use db/structure.sql with SQL as schema format
+    # This is needed to store in the schema SQL statements not covered by the ORM
+    config.active_record.schema_format = :sql
   end
 end

config/initializers/elasticsearch.rb

@@ -0,0 +1,32 @@
+class AddTsvectorColumnToPost < ActiveRecord::Migration
+  def up
+    execute <<-SQL
+      ALTER TABLE posts ADD COLUMN tsv tsvector;
+
+      CREATE FUNCTION posts_trigger() RETURNS trigger AS $$
+      begin
+        new.tsv :=
+          to_tsvector('simple', unaccent(coalesce(new.title::text, ''))) ||
+          to_tsvector('simple', unaccent(coalesce(new.description::text, ''))) ||
+          to_tsvector('simple', unaccent(coalesce(new.tags::text, '')));
+        return new;
+      end
+      $$ LANGUAGE plpgsql;
+
+      CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE
+      ON posts FOR EACH ROW EXECUTE PROCEDURE posts_trigger();
+    SQL
+
+    add_index :posts, :tsv, using: "gin"
+  end
+
+  def down
+    execute <<-SQL
+      DROP TRIGGER tsvectorupdate ON posts;
+      DROP FUNCTION posts_trigger();
+    SQL
+
+    remove_index :posts, :tsv
+    remove_column :posts, :tsv
+  end
+end

db/migrate/20190523213421_add_tsvector_column_to_post.rb

@@ -0,0 +1,9 @@
+class EnableUnaccentExtension < ActiveRecord::Migration
+  def up
+    enable_extension "unaccent"
+  end
+
+  def down
+    disable_extension "unaccent"
+  end
+end