Store Alchemy::Page as a single PgSearch::Document

Previously the page and the ingredients (and in previous versions also essences) were stored as separate PgSearch::Document entries. On search these document were combined to a single document to make it usable in the search itself. These mechanic made it pretty difficult to extend the search with other models. In newer versions of Alchemy the page is only available after a new page version is released and these mechanic is used to create a single PgSearch::Document with the content of the page and all supported ingredients. This change makes the whole search and the index creation less complex.

The Alchemy module now has searchable_ingredients attribute which makes it easier to change the indexing from the outside. Furthermore a few extensions are now in a Search namespace, because they are not related to the PgSearch gem.

app/extensions/alchemy/pg_search/pg_search_document_extension.rb

@@ -2,20 +2,6 @@ module Alchemy::PgSearch::PgSearchDocumentExtension
   def self.prepended(base)
     base.belongs_to :page, class_name: "::Alchemy::Page", foreign_key: "page_id", optional: true
   end
-
-  ##
-  # get a list of excerpts of the searched phrase
-  # The JSON_AGG - method will transform the grouped content entries into json which have to be "unpacked".
-  # @return [array<string>]
-  def excerpts
-    return [] if content.blank?
-    begin
-      parsed_content = JSON.parse content
-      parsed_content.kind_of?(Array) ? parsed_content : []
-    rescue JSON::ParserError
-      []
-    end
-  end
 end
 
 PgSearch::Document.prepend(Alchemy::PgSearch::PgSearchDocumentExtension)

app/extensions/alchemy/pg_search/element_extension.rb → app/extensions/alchemy/search/element_extension.rb

@@ -1,4 +1,4 @@
-module Alchemy::PgSearch::ElementExtension
+module Alchemy::Search::ElementExtension
   def self.prepended(base)
     base.attr_writer :searchable
   end
@@ -12,4 +12,4 @@ def searchable?
   end
 end
 
-Alchemy::Element.prepend(Alchemy::PgSearch::ElementExtension)
+Alchemy::Element.prepend(Alchemy::Search::ElementExtension)

app/extensions/alchemy/search/ingredient_extension.rb

@@ -0,0 +1,14 @@
+module Alchemy::Search::IngredientExtension
+  def searchable_content
+    send(Alchemy.searchable_ingredients[type.to_sym])&.squish
+  end
+
+  def searchable?
+    Alchemy.searchable_ingredients.has_key?(type.to_sym) &&
+      (definition.key?(:searchable) ? definition[:searchable] : true) &&
+      !!element&.searchable?
+  end
+end
+
+# add the PgSearch model to all ingredients
+Alchemy::Ingredient.prepend(Alchemy::Search::IngredientExtension)

app/extensions/alchemy/search/page_extension.rb

@@ -0,0 +1,10 @@
+# Enable Postgresql full text indexing.
+#
+module Alchemy::Search::PageExtension
+  def searchable?
+    (definition.key?(:searchable) ? definition[:searchable] : true) &&
+      searchable && public? && !layoutpage?
+  end
+end
+
+Alchemy::Page.prepend(Alchemy::Search::PageExtension)

app/views/alchemy/search/_result.html.erb

@@ -1,12 +1,6 @@
 <li class="search_result">
-  <% page = result.page %>
+  <% page = result.searchable %>
   <h3><%= link_to page.name, show_alchemy_page_path(page) %></h3>
-  <% if result.excerpts.any? %>
-    <% result.excerpts.each do |excerpt| %>
-    <p><%= highlighted_excerpt(excerpt, params[:query]) %></p>
-    <% end %>
-  <% else %>
-    <p><%= page.meta_description %></p>
-  <% end %>
+  <p><%= highlighted_excerpt(result.content, params[:query]) %></p>
   <p><%= link_to page.urlname, show_alchemy_page_path(page) %></p>
 </li>

lib/alchemy-pg_search.rb

@@ -5,24 +5,22 @@ module Alchemy
   mattr_accessor :search_class
   @@search_class = PgSearch
 
-  module PgSearch
-    SEARCHABLE_INGREDIENTS = %w[Text Richtext Picture]
+  mattr_accessor :searchable_ingredients
+  @@searchable_ingredients = {
+    "Alchemy::Ingredients::Text": :value,
+    "Alchemy::Ingredients::Richtext": :stripped_body,
+    "Alchemy::Ingredients::Picture": :caption,
+  }
 
+  module PgSearch
     extend Config
 
     ##
-    # is ingredient searchable?
-    # @param ingredient_type [string]
-    # @return [boolean]
-    def self.is_searchable?(ingredient_type)
-      SEARCHABLE_INGREDIENTS.include?(ingredient_type.gsub(/Alchemy::Ingredients::/, ""))
-    end
-
-    ##
-    # index all supported Alchemy models
+    # index all supported Alchemy pages
     def self.rebuild
-      [Alchemy::Page, Alchemy::Ingredient].each do |model|
-        ::PgSearch::Multisearch.rebuild(model)
+      ActiveRecord::Base.transaction do
+        ::PgSearch::Document.delete_all
+        Alchemy::Page.all.each{ |page| index_page(page) }
       end
     end
 
@@ -39,14 +37,16 @@ def self.remove_page(page)
     #
     # @param page [Alchemy::Page]
     def self.index_page(page)
-      remove_page page
-
       page.update_pg_search_document
-      page.all_elements.includes(:ingredients).find_each do |element|
-        element.ingredients.select { |i| Alchemy::PgSearch.is_searchable?(i.type) }.each do |ingredient|
-          ingredient.update_pg_search_document
-        end
-      end
+
+      document = page.pg_search_document
+      return if document.nil?
+
+      ingredient_content = page.all_elements.includes(ingredients: {element: :page}).map do |element|
+        element.ingredients.select { |i| i.searchable? }.map(&:searchable_content).join(" ")
+      end.join(" ")
+
+      document.update_column(:content, "#{document.content} #{ingredient_content}".squish)
     end
 
     ##
@@ -56,13 +56,17 @@ def self.index_page(page)
     # @param ability [nil|CanCan::Ability]
     # @return [ActiveRecord::Relation]
     def self.search(query, ability: nil)
-      query = ::PgSearch.multisearch(query)
-                        .select("JSON_AGG(content) as content", :page_id)
-                        .reorder("")
-                        .group(:page_id)
-                        .joins(:page)
+      query = ::PgSearch.multisearch(query).includes(:searchable)
 
-      query = query.merge(Alchemy::Page.accessible_by(ability, :read)) if ability
+      if ability
+        # left_joins method is not usable here, because the order of the joins are incorrect
+        # and would result in a SQL error. We can receive the correct query order with these
+        # odd left join string
+        # Ref: https://guides.rubyonrails.org/active_record_querying.html#using-a-string-sql-fragment
+        query = query
+                  .joins("LEFT JOIN alchemy_pages ON alchemy_pages.id = pg_search_documents.page_id")
+                  .merge(Alchemy::Page.accessible_by(ability, :read))
+      end
 
       query
     end

spec/dummy/config/alchemy/elements.yml

@@ -34,11 +34,11 @@
       searchable: false
     - role: public
       type: Richtext
-      default: "This is some public text."
+      default: "This is some <i>public</i> richtext."
     - role: confidential
       type: Richtext
       searchable: false
-      default: "This is some confidential text."
+      default: "This is some <i>confidential</i> richtext."
     - role: image
       type: Picture
     - role: secret_image

spec/dummy/config/alchemy/page_layouts.yml

@@ -6,6 +6,12 @@
   - article
   - secrets
 
+- name: mixed
+  elements:
+    - mixed
+  autogenerate:
+    - mixed
+
 - name: search
   searchresults: true
   unique: true

spec/features/fulltext_search_feature_spec.rb

@@ -21,15 +21,19 @@
       image.save
     end
 
+    before do
+      Alchemy::PgSearch.rebuild
+    end
+
     it "displays search results from text ingredients" do
-      visit("/suche?query=search")
+      visit("/suche?query=headline")
       within(".search_results") do
         expect(page).to have_content("This is a headline everybody should be able to search for.")
       end
     end
 
-    it "displays search results from richtext essences" do
-      visit("/suche?query=search")
+    it "displays search results from richtext ingredient" do
+      visit("/suche?query=text%20block")
       within(".search_results") do
         expect(page).to have_content("This is a text block everybody should be able to search for.")
       end
@@ -59,7 +63,6 @@
     it "does not display results placed on global pages" do
       # A layout page is configured and the page is indexed after publish
       public_page.update!(layoutpage: true)
-      Alchemy::PgSearch.index_page public_page
 
       visit("/suche?query=search")
       expect(page).to have_css("h2.no_search_results")
@@ -131,6 +134,7 @@
 
       before do
         nested_element.ingredient_by_role("headline").update!({ value: "Content from nested element" })
+        Alchemy::PgSearch.rebuild
       end
 
       it "displays search results from nested elements" do
@@ -184,6 +188,7 @@
           page_version: create(:alchemy_page, :public).public_version,
         )
       end
+      Alchemy::PgSearch.rebuild
     end
 
     context "when default config is used" do