Topics (#89)

* Fixing a name typo

* better prompt fetch from db

* Fixing the Trix UI

* Few gem updates;

* Upgrading to Rails 7.0.5.1

* Fixing an association issue in between organization and users

* fixing a typo organiation.members

* fixing the prompt table

* Upgradig a few gems

* Upgrading to Rails 7.0.6

* Upgrading ActionText to version 7.0.6

* fixing the prompt table

* Aligning Gemfile.loc

* Upgrading to Rails 7.0.6

* Upgrading groupdate, faraday, rails-dom-testig and few other gems

* Upgrading honeybadger to version 6.1.3

* Minor gem upgrades

* Renamed and improved the prompt import function

* Adding octokit

* schema.rb now reflects the correct database structure

* Middle of the gate

* Upgrading to Flowbite 1.7.0

* Extracting Topic from question

---------

Signed-off-by: spaquet <176050+spaquet@users.noreply.github.com>

Gemfile

@@ -118,6 +118,9 @@ gem 'image_processing', '~> 1.2'
 gem 'ruby-vips', '>= 2.1.0'
 gem 'aws-sdk-s3', require: false
 
+# Octokig [https://github.com/octokit/octokit.rb]
+# Mostly used to import openAI prompts from GitHub repo using the import_prompts.rake task
+gem 'octokit', '~> 6.1.1'
 
 # Generate QRCode [https://github.com/whomwah/rqrcode]
 gem 'rqrcode', '~> 2.0'

Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/faker-ruby/faker.git
-  revision: 4558f1bb9bb80c13b2ff0e3fcad78ac02c7f5515
+  revision: a342e3e2200ca7862ef30e8e54bddc215dbd1ff8
   specs:
     faker (3.2.0)
       i18n (>= 1.8.11, < 2)
@@ -158,7 +158,7 @@ GEM
     down (5.4.1)
       addressable (~> 2.8)
     erubi (1.12.0)
-    faraday (2.7.8)
+    faraday (2.7.9)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-multipart (1.0.4)
@@ -170,8 +170,8 @@ GEM
       rake
     globalid (1.1.0)
       activesupport (>= 5.0)
-    groupdate (6.2.1)
-      activesupport (>= 5.2)
+    groupdate (6.3.0)
+      activesupport (>= 6.1)
     hashie (5.0.0)
     honeybadger (5.2.1)
     htmlentities (4.3.4)
@@ -191,7 +191,7 @@ GEM
     invisible_captcha (2.1.0)
       rails (>= 5.2)
     io-console (0.6.0)
-    irb (1.7.0)
+    irb (1.7.1)
       reline (>= 0.3.0)
     jbuilder (2.11.5)
       actionview (>= 5.0.0)
@@ -249,6 +249,9 @@ GEM
       rack (>= 1.2, < 4)
       snaky_hash (~> 2.0)
       version_gem (~> 1.1)
+    octokit (6.1.1)
+      faraday (>= 1, < 3)
+      sawyer (~> 0.9)
     oj (3.15.0)
     omniauth (2.1.1)
       hashie (>= 3.4.6)
@@ -302,8 +305,9 @@ GEM
       activesupport (= 7.0.6)
       bundler (>= 1.15.0)
       railties (= 7.0.6)
-    rails-dom-testing (2.0.3)
-      activesupport (>= 4.2.0)
+    rails-dom-testing (2.1.1)
+      activesupport (>= 5.0.0)
+      minitest
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.6.0)
       loofah (~> 2.21)
@@ -334,7 +338,7 @@ GEM
       chunky_png (~> 1.0)
       rqrcode_core (~> 1.0)
     rqrcode_core (1.2.0)
-    rubocop (1.53.1)
+    rubocop (1.54.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -361,6 +365,9 @@ GEM
     ruby_http_client (3.5.5)
     rubyzip (2.3.2)
     safely_block (0.4.0)
+    sawyer (0.9.2)
+      addressable (>= 2.3.5)
+      faraday (>= 0.17.3, < 3)
     selenium-webdriver (4.10.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
@@ -467,6 +474,7 @@ DEPENDENCIES
   merit
   nokogiri (~> 1.15.0)
   noticed (~> 1.5)
+  octokit (~> 6.1.1)
   oj (~> 3.15.0)
   omniauth (~> 2.1.0)
   omniauth-google-oauth2 (~> 1.1.1)

app/models/prompt.rb

@@ -3,11 +3,12 @@
 # Table name: prompts
 #
 #  id              :uuid             not null, primary key
-#  function        :string
+#  function_call   :string
+#  functions       :text
 #  label           :string(50)       not null
-#  model           :string           default("gpt-3.5"), not null
-#  prompt          :text             not null
-#  title           :string(50)       not null
+#  messages        :text             not null
+#  model           :string
+#  title           :string(150)      not null
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
 #  organization_id :uuid
@@ -19,13 +20,28 @@
 #  index_prompts_on_organization_id            (organization_id)
 #
 class Prompt < ApplicationRecord
-  validates :label, presence: true, length: { minimum: 3, maximum: 50 }, uniqueness: { case_sensitive: false }
-  validates :title, presence: true, length: { minimum: 3, maximum: 50 }
+  validates :label,    presence: true, length: { minimum: 3, maximum: 50 }, uniqueness: { case_sensitive: false }
+  validates :title,    presence: true, length: { minimum: 3, maximum: 150 }
+  validates :messages, presence: true
 
   # Prompts table is used to saved prompts that are used by AI to extract critical information.
   # labels is the name of the prompt
   # title is a short description of the prompts
-  # prompt is the prompt ;-)
-  # some organization may require specific prompts
+  # message is formatted as openAI message
+
+  # Retrieves the messages with dynamic parameter replacement
+  #
+  # @param params [Hash] The parameters used to replace placeholders in the message
+  # @return [String] The message with placeholders replaced by corresponding parameter values
+  def get_messages(params = {})
+    replaced_messages = messages.dup
+
+    params.each do |key, value|
+      escaped_value = value.to_s.gsub(/["\\]/, '\\\\\0')
+      replaced_messages.gsub!("%{#{key}}", escaped_value)
+    end
+
+    replaced_messages
+  end  
 
 end

app/models/topic.rb

@@ -14,11 +14,12 @@
 #
 # Indexes
 #
-#  index_topics_on_event_id     (event_id)
-#  index_topics_on_name         (name)
-#  index_topics_on_question_id  (question_id)
-#  index_topics_on_room_id      (room_id)
-#  index_topics_on_user_id      (user_id)
+#  index_topics_on_event_id                             (event_id)
+#  index_topics_on_name                                 (name)
+#  index_topics_on_question_id                          (question_id)
+#  index_topics_on_question_id_and_room_id_and_user_id  (question_id,room_id,user_id) UNIQUE
+#  index_topics_on_room_id                              (room_id)
+#  index_topics_on_user_id                              (user_id)
 #
 class Topic < ApplicationRecord
 end

app/services/prompt_retriever_service.rb

@@ -6,7 +6,7 @@ module PromptRetrieverService
   # @param label [String] The label of the prompt to retrieve
   # @param organization_id [String, nil] The organization ID (optional)
   # @return [Hash, nil] A hash with :title, :prompt, :model, and :function if a prompt is found, nil otherwise
-  def self.retrieve(label, organization_id = nil)
+  def self.retrieve(label, organization_id = nil, params = {})
     prompt = Prompt.find_by(label: label, organization_id: organization_id)
 
     # Fallback in case we have no prompt when passing a label AND an organization_id
@@ -16,12 +16,20 @@ def self.retrieve(label, organization_id = nil)
 
     return unless prompt
 
-    {
+    result = {
       title: prompt.title,
-      prompt: prompt.prompt,
+      messages: prompt.messages,
       model: prompt.model,
-      function: prompt.function || nil
+      functions: prompt.functions || nil,
+      function_call: prompt.function_call || nil
     }
+
+    if params 
+      result[:messages] = prompt.get_messages(params)
+    end
+
+    result
+
   end
 
 end

app/sidekiq/event_topics_extraction_job.rb

@@ -0,0 +1,16 @@
+class EventTopicsExtractionJob
+  include Sidekiq::Job
+
+  require "openai"
+
+  def perform(event)
+    # Initialize an openAI client
+    client = OpenAI::Client.new
+
+    # Fetch all the questions from an event
+
+    # Aggregate the questions in one document
+
+    # Extract the topics from the questions with openAI
+  end
+end

app/sidekiq/question_keywords_extraction_job.rb

@@ -2,6 +2,7 @@ class QuestionKeywordsExtractionJob
   include Sidekiq::Job
 
   require "openai"
+  require "yaml"
   require_relative "../services/prompt_retriever_service.rb"
 
   def perform(question)
@@ -11,20 +12,28 @@ def perform(question)
     # Parse the question
     qh = JSON.parse(question)
 
-    # Use GTP 4 Chat to extract the relevant keywords from the question
-    pr = PromptRetrieverService.retrieve("question-keyword-extraction", nil)
-    puts "## PROMPT ##"
-    puts pr
+    question_title = qh.dig("title")
+    # Use Chat GPT to extract the relevant keywords from the question
+    pr = PromptRetrieverService.retrieve("question-keyword-extraction", nil, {title: question_title})
+    # puts "## PROMPT ##"
+    # puts pr.inspect
+
+    messages = JSON.parse(pr[:messages])
+
+    params = {
+      model: pr[:model],
+      messages: messages,
+      temperature: 0.7,
+      user: qh.dig("user_id")
+    }
+
+    # puts params
+
     response = client.chat(
-      parameters: {
-          model: "gpt-4",
-          messages: [
-            { role: "system", content: pr[:prompt] },
-            { role: "user", content: "Extract keywords from this text:\n\n" + qh.dig("title")}
-          ],
-          temperature: 0.7,
-          user: qh.dig("user_id")
-      })
+      parameters: params
+      )
+
+    # puts response
     keywords = response.dig("choices", 0, "message","content")
     keywords = keywords.split(", ")
     keywords.map{ |word| word.strip.chomp(".") }

app/sidekiq/question_processing_job.rb

@@ -1,14 +1,21 @@
+# app/sidekiq/question_processing_job.rb
 class QuestionProcessingJob
   include Sidekiq::Job
 
   def perform(question)
-    # Extract key elements from the question such as Tone, Keywords
-    # Topics are extracted via a cron job
-
-    # Extract Tone from the quesstion
+    # Extract Tone from the question
     QuestionToneJob.perform_async(question)
-
-    # Extract Keywords from the question
-    QuestionKeywordsExtractionJob.perform_async(question)
+
+    # Extract the status from the question JSON
+    question_status = JSON.parse(question)['status']
+
+    # Check if the question status is not "rejected"
+    if question_status != "rejected"
+      # Extract Keywords from the question
+      QuestionKeywordsExtractionJob.perform_async(question)
+
+      # Extract Topic from the question
+      QuestionTopicExtractionJob.perform_async(question)
+    end
   end
 end

app/sidekiq/question_topic_extraction_job.rb

@@ -0,0 +1,71 @@
+# app/sidekiq/question_topic_extraction_job.rb
+
+# Description: used to extract the topic covered by a question.
+
+class QuestionTopicExtractionJob
+  include Sidekiq::Job
+
+  require "openai"
+
+  def perform(question)
+    # Initialize an openAI client
+    client = OpenAI::Client.new
+
+    # Parse the question
+    qh = JSON.parse(question)
+    # puts "## Question ##"
+    # puts qh
+
+    # Extracting key values from the question
+    room_id = qh.dig("room_id")
+    question_id = qh.dig("id")
+    question_title = qh.dig("title")
+    user_id = qh.dig("user_id")
+
+    # Find the event id
+    event_id = Room.select(:event_id).find(room_id).event_id
+    puts "## Event id: #{event_id}"
+
+    # Extracting the topic from the question using openAI
+    pr = PromptRetrieverService.retrieve("question-topic-extraction", nil, {title: question_title})
+    # puts "## PROMPT ##"
+    # puts pr.inspect
+
+    messages = JSON.parse(pr[:messages])
+
+    params = {
+      model: pr[:model],
+      messages: messages,
+      temperature: 0.7,
+      user: qh.dig("user_id")
+    }
+
+    # puts params
+
+    response = client.chat(
+      parameters: params
+      )
+
+    # puts response
+
+    topics = response.dig("choices", 0, "message","content")
+    topics = topics.split(", ")
+    topics.map{ |word| word.strip.chomp(".") }
+    puts "TOPICS IS ARRAY? #{topics.kind_of?(Array)}"
+    puts "QUESTION: #{question_title}"
+    puts "TOPICS AS ARRAY: #{topics}"
+    if topics.kind_of?(Array)
+      topics.each do |topic|
+        param = {
+          event_id: event_id,
+          question_id: question_id,
+          room_id: room_id,
+          user_id: user_id,
+          description: "Question: #{question_title}",
+          name: topic
+        }
+        Topic.upsert(param , unique_by: [:question_id, :room_id, :user_id])
+      end
+    end
+  end
+end

db/migrate/20230514234427_create_prompts.rb

@@ -3,10 +3,11 @@ def change
     create_table :prompts, id: :uuid do |t|
       t.uuid    :organization_id
       t.string  :label,          null: false, limit: 50
-      t.string  :title,          null: false, limit: 50
+      t.string  :title,          null: false, limit: 150
       t.string  :model
-      t.text    :message,        null: false
-      t.text    :function
+      t.text    :messages,       null: false
+      t.text    :functions
+      t.string  :function_call
 
       t.timestamps
     end