Rag/rag transition 3

lib/chatbot/rag.ex

@@ -7,23 +7,40 @@
 
   @provider Ai.Nx.new(%{embeddings_serving: Rag.EmbeddingServing})
 
-  def ingest(path) do
-    path
-    |> load()
-    |> index()
+  def ingest_ecto() do
+    docs_url = "https://repo.hex.pm/docs/ecto-3.12.5.tar.gz"
+
+    code_url = "https://repo.hex.pm/tarballs/ecto-3.12.5.tar"
+
+    req = Req.new(url: docs_url) |> ReqHex.attach()
+    docs_tarball = Req.get!(req).body
+
+    docs =
+      for {file, content} <- docs_tarball, text_file?(file) do
+        file = to_string(file)
+        %{source: file, document: content}
+      end
+
+    req = Req.new(url: code_url) |> ReqHex.attach()
+    code_tarball = Req.get!(req).body
+
+    code =
+      for {file, content} <- code_tarball["contents.tar.gz"] do
+        %{source: file, document: content}
+      end
+
+    index(docs ++ code)
   end
 
-  def load(path) do
-    path
-    |> list_text_files()
-    |> Enum.map(&%{source: &1})
-    |> Enum.map(&Rag.Loading.load_file(&1))
+  defp text_file?(file) when is_list(file) do
+    file
+    |> to_string()
+    |> String.ends_with?([".html", ".md", ".txt"])
   end
 
-  defp list_text_files(path) do
-    path
-    |> Path.join("/**/*.txt")
-    |> Path.wildcard()
+  defp text_file?(file) when is_binary(file) do
+    file
+    |> String.ends_with?([".html", ".md", ".txt"])
   end
 
   def index(ingestions) do
@@ -46,9 +63,9 @@
     Enum.map(chunks, &Map.put(ingestion, :chunk, &1.text))
   end
 
   def query(query) do
     generation =
       Generation.new(query)
       |> Embedding.generate_embedding(@provider)
       |> Retrieval.retrieve(:fulltext_results, fn generation -> query_fulltext(generation) end)
       |> Retrieval.retrieve(:semantic_results, fn generation ->
@@ -85,7 +102,7 @@
     |> Map.put_new(:updated_at, now)
   end
 
   defp query_with_pgvector(%{query_embedding: query_embedding}, limit \\ 3) do
     {:ok,
      Repo.all(
        from(c in Chatbot.Rag.Chunk,
@@ -95,7 +112,7 @@
      )}
   end
 
   defp query_fulltext(%{query: query}, limit \\ 3) do
     query = query |> String.trim() |> String.replace(" ", " & ")
 
     {:ok,
@@ -107,7 +124,7 @@
      )}
   end
 
   defp smollm_prompt(query, context) do
     """
     <|im_start|>system
     You are a helpful assistant.<|im_end|>

lib/chatbot/rag/serving.ex

@@ -1,8 +1,12 @@
 defmodule Chatbot.Rag.Serving do
   def build_embedding_serving() do
-    repo = {:hf, "thenlper/gte-small"}
+    repo = {:hf, "jinaai/jina-embeddings-v2-base-code"}
 
-    {:ok, model_info} = Bumblebee.load_model(repo)
+    {:ok, model_info} =
+      Bumblebee.load_model(repo,
+        params_filename: "model.safetensors",
+        spec_overrides: [architecture: :base]
+      )
 
     {:ok, tokenizer} = Bumblebee.load_tokenizer(repo)
 

mix.exs

@@ -33,10 +33,11 @@ defmodule Chatbot.MixProject do
   # Type `mix help deps` for examples and options.
   defp deps do
     [
+      {:req_hex, "~> 0.2.1"},
       {:pgvector, "~> 0.3.0"},
       {:ecto, "~> 3.0"},
       {:exla, "~> 0.9.1"},
-      {:bumblebee, "~> 0.6.0"},
+      {:bumblebee, github: "joelpaulkoch/bumblebee", branch: "jina-embeddings-v2-base-code"},
       {:text_chunker, "~> 0.3.1"},
       {:ex_machina, "~> 2.8"},
       {:bitcrowd_ecto, "~> 1.0"},

mix.lock

@@ -4,13 +4,13 @@
   "bandit": {:hex, :bandit, "1.5.7", "6856b1e1df4f2b0cb3df1377eab7891bec2da6a7fd69dc78594ad3e152363a50", [:mix], [{:hpax, "~> 1.0.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "f2dd92ae87d2cbea2fa9aa1652db157b6cba6c405cb44d4f6dd87abba41371cd"},
   "bitcrowd_ecto": {:hex, :bitcrowd_ecto, "1.0.0", "b255cf7b8e22bc17adeb8bbc9907ef02dcdc751fd68ab3e444b0a098dac99b65", [:mix], [{:ecto, "~> 3.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.6", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ex_money, "~> 5.12", [hex: :ex_money, repo: "hexpm", optional: true]}, {:jason, "> 0.0.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "862effc5eba425176472c2f01dd50b9994622089fa4bc2a11d75732b47dcdab9"},
   "bitstyles_phoenix": {:hex, :bitstyles_phoenix, "2.5.0", "c02aae26fcf6ff752738b35aa97f3991c67533412eda381abaad3e22aa2e2215", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.18.12 or ~> 0.19.0 or ~> 0.20.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "fe018f0eeec6c84afedc1785495efe0a651ecf34656c4b52f6da09d5149a9f28"},
-  "bumblebee": {:hex, :bumblebee, "0.6.0", "1c731313308ff9fde2effc4a2f366742dbd78e227b84e980eb2804f6b9281724", [:mix], [{:axon, "~> 0.7.0", [hex: :axon, repo: "hexpm", optional: false]}, {:jason, "~> 1.4.0", [hex: :jason, repo: "hexpm", optional: false]}, {:nx, "~> 0.9.0", [hex: :nx, repo: "hexpm", optional: false]}, {:nx_image, "~> 0.1.0", [hex: :nx_image, repo: "hexpm", optional: false]}, {:nx_signal, "~> 0.2.0", [hex: :nx_signal, repo: "hexpm", optional: false]}, {:progress_bar, "~> 3.0", [hex: :progress_bar, repo: "hexpm", optional: false]}, {:safetensors, "~> 0.1.3", [hex: :safetensors, repo: "hexpm", optional: false]}, {:tokenizers, "~> 0.4", [hex: :tokenizers, repo: "hexpm", optional: false]}, {:unpickler, "~> 0.1.0", [hex: :unpickler, repo: "hexpm", optional: false]}, {:unzip, "~> 0.12.0", [hex: :unzip, repo: "hexpm", optional: false]}], "hexpm", "a8b863179d314e9615b00291d5dcd2dc043b294edc25b4483d5c88d1c8d21c89"},
+  "bumblebee": {:git, "https://github.com/joelpaulkoch/bumblebee.git", "1c27550b1cbfcfff3b0fb99e84bf2b3d5ccb4369", [branch: "jina-embeddings-v2-base-code"]},
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
-  "castore": {:hex, :castore, "1.0.9", "5cc77474afadf02c7c017823f460a17daa7908e991b0cc917febc90e466a375c", [:mix], [], "hexpm", "5ea956504f1ba6f2b4eb707061d8e17870de2bee95fb59d512872c2ef06925e7"},
+  "castore": {:hex, :castore, "1.0.12", "053f0e32700cbec356280c0e835df425a3be4bc1e0627b714330ad9d0f05497f", [:mix], [], "hexpm", "3dca286b2186055ba0c9449b4e95b97bf1b57b47c1f2644555879e659960c224"},
   "complex": {:hex, :complex, "0.6.0", "b0130086a7a8c33574d293b2e0e250f4685580418eac52a5658a4bd148f3ccf1", [:mix], [], "hexpm", "0a5fa95580dcaf30fcd60fe1aaf24327c0fe401e98c24d892e172e79498269f9"},
   "credo": {:hex, :credo, "1.7.10", "6e64fe59be8da5e30a1b96273b247b5cf1cc9e336b5fd66302a64b25749ad44d", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "71fbc9a6b8be21d993deca85bf151df023a3097b01e09a2809d460348561d8cd"},
   "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"},
-  "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
+  "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"},
   "dialyxir": {:hex, :dialyxir, "1.4.4", "fb3ce8741edeaea59c9ae84d5cec75da00fa89fe401c72d6e047d11a61f65f70", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "cd6111e8017ccd563e65621a4d9a4a1c5cd333df30cebc7face8029cacb4eff6"},
   "dns_cluster": {:hex, :dns_cluster, "0.1.3", "0bc20a2c88ed6cc494f2964075c359f8c2d00e1bf25518a6a6c7fd277c9b0c66", [:mix], [], "hexpm", "46cb7c4a1b3e52c7ad4cbe33ca5079fbde4840dedeafca2baf77996c2da1bc33"},
   "earmark": {:hex, :earmark, "1.4.47", "7e7596b84fe4ebeb8751e14cbaeaf4d7a0237708f2ce43630cfd9065551f94ca", [:mix], [], "hexpm", "3e96bebea2c2d95f3b346a7ff22285bc68a99fbabdad9b655aa9c6be06c698f8"},
@@ -27,6 +27,7 @@
   "floki": {:hex, :floki, "0.36.2", "a7da0193538c93f937714a6704369711998a51a6164a222d710ebd54020aa7a3", [:mix], [], "hexpm", "a8766c0bc92f074e5cb36c4f9961982eda84c5d2b8e979ca67f5c268ec8ed580"},
   "gettext": {:hex, :gettext, "0.26.1", "38e14ea5dcf962d1fc9f361b63ea07c0ce715a8ef1f9e82d3dfb8e67e0416715", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "01ce56f188b9dc28780a52783d6529ad2bc7124f9744e571e1ee4ea88bf08734"},
   "glob_ex": {:hex, :glob_ex, "0.1.11", "cb50d3f1ef53f6ca04d6252c7fde09fd7a1cf63387714fe96f340a1349e62c93", [:mix], [], "hexpm", "342729363056e3145e61766b416769984c329e4378f1d558b63e341020525de4"},
+  "hex_core": {:hex, :hex_core, "0.10.3", "7dc866ca8f5830566aacef141bdde4f1d61f687407c7070611c349e983f185d2", [:rebar3], [], "hexpm", "bc7c2b6ceb99e29550f74234952ad3b8153e6f2dbc38892bd0ff46043be073f5"},
   "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"},
   "igniter": {:hex, :igniter, "0.5.33", "799a49a8eb7e2fbebd6af2b770a856fa8ede9acad73a74269a04b8e775ead199", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:inflex, "~> 2.0", [hex: :inflex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:owl, "~> 0.11", [hex: :owl, repo: "hexpm", optional: false]}, {:phx_new, "~> 1.7", [hex: :phx_new, repo: "hexpm", optional: true]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:rewrite, ">= 1.1.1 and < 2.0.0-0", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "ac962445d426dd3815e6b5568daa86586487293d126a946a67d9cf17d0665005"},
   "inflex": {:hex, :inflex, "2.1.0", "a365cf0821a9dacb65067abd95008ca1b0bb7dcdd85ae59965deef2aa062924c", [:mix], [], "hexpm", "14c17d05db4ee9b6d319b0bff1bdf22aa389a25398d1952c7a0b5f3d93162dd8"},
@@ -56,6 +57,7 @@
   "progress_bar": {:hex, :progress_bar, "3.0.0", "f54ff038c2ac540cfbb4c2bfe97c75e7116ead044f3c2b10c9f212452194b5cd", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "6981c2b25ab24aecc91a2dc46623658e1399c21a2ae24db986b90d678530f2b7"},
   "rag": {:git, "https://github.com/bitcrowd/rag.git", "d59d25e997aff214aff3e507cad00b9d537d4e68", []},
   "req": {:hex, :req, "0.5.6", "8fe1eead4a085510fe3d51ad854ca8f20a622aae46e97b302f499dfb84f726ac", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cfaa8e720945d46654853de39d368f40362c2641c4b2153c886418914b372185"},
+  "req_hex": {:hex, :req_hex, "0.2.1", "f7566a61afde1f3f31977fdb230c2b9e801742da5de0b66a7b1e14febb304be7", [:mix], [{:hex_core, "~> 0.10.0", [hex: :hex_core, repo: "hexpm", optional: false]}, {:req, "~> 0.4.0 or ~> 0.5.0", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "f132012071714cfc277ff52ea5a80b793a4bebd79675d75820e02b425d897461"},
   "rewrite": {:hex, :rewrite, "1.1.2", "f5a5d10f5fed1491a6ff48e078d4585882695962ccc9e6c779bae025d1f92eda", [:mix], [{:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.0", [hex: :sourceror, repo: "hexpm", optional: false]}, {:text_diff, "~> 0.1", [hex: :text_diff, repo: "hexpm", optional: false]}], "hexpm", "7f8b94b1e3528d0a47b3e8b7bfeca559d2948a65fa7418a9ad7d7712703d39d4"},
   "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.2", "5f25cbe220a8fac3e7ad62e6f950fcdca5a5a5f8501835d2823e8c74bf4268d5", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "63d1bd5f8e23096d1ff851839923162096364bac8656a4a3c00d1fff8e83ee0a"},
   "safetensors": {:hex, :safetensors, "0.1.3", "7ff3c22391e213289c713898481d492c9c28a49ab1d0705b72630fb8360426b2", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:nx, "~> 0.5", [hex: :nx, repo: "hexpm", optional: false]}], "hexpm", "fe50b53ea59fde4e723dd1a2e31cfdc6013e69343afac84c6be86d6d7c562c14"},

priv/repo/migrations/20250311215959_create_chunks_table.exs

@@ -10,7 +10,7 @@ defmodule Chatbot.Repo.Migrations.CreateChunksTable do
       add(:document, :text)
       add(:source, :text)
       add(:chunk, :text)
-      add(:embedding, :vector, size: 384)
+      add(:embedding, :vector, size: 768)
 
       timestamps()
     end