Merge pull request #65 from janhq/37-api-with-llamacpp-for-embedding

37 api with llamacpp for embedding

Author: tikikun

config.json

@@ -8,6 +8,7 @@
 	"custom_config": {
 		"llama_model_path": "/Users/alandao/Documents/codes/nitro.cpp_temp/models/llama2_7b_chat_uncensored.Q4_0.gguf",
 		"ctx_len": 2048,
-		"ngl": 100
+		"ngl": 100,
+		"embedding":true
 	}
 }

controllers/llamaCPP.cc

@@ -3,8 +3,11 @@
 #include "nitro_utils.h"
 #include <chrono>
 #include <cstring>
-#include <thread>
+#include <drogon/HttpResponse.h>
 #include <regex>
+#include <thread>
+
+using namespace inferences;
 
 std::string create_return_json(const std::string &id, const std::string &model,
                                const std::string &content,
@@ -35,7 +38,7 @@ std::string create_return_json(const std::string &id, const std::string &model,
   return Json::writeString(writer, root);
 }
 
-void llamaCPP::asyncHandleHttpRequest(
+void llamaCPP::chatCompletion(
     const HttpRequestPtr &req,
     std::function<void(const HttpResponsePtr &)> &&callback) {
   const auto &jsonBody = req->getJsonObject();
@@ -196,3 +199,29 @@ void llamaCPP::asyncHandleHttpRequest(
                                                       "chat_completions.txt");
   callback(resp);
 }
+
+void llamaCPP::embedding(
+    const HttpRequestPtr &req,
+    std::function<void(const HttpResponsePtr &)> &&callback) {
+  auto lock = llama.lock();
+
+  const auto &jsonBody = req->getJsonObject();
+
+  llama.rewind();
+  llama_reset_timings(llama.ctx);
+  if (jsonBody->isMember("content") != 0) {
+    llama.prompt = (*jsonBody)["content"].asString();
+  } else {
+    llama.prompt = "";
+  }
+  llama.params.n_predict = 0;
+  llama.loadPrompt();
+  llama.beginCompletion();
+  llama.doCompletion();
+
+  const json data = format_embedding_response(llama);
+  auto resp = drogon::HttpResponse::newHttpResponse();
+  resp->setBody(data.dump());
+  resp->setContentTypeString("application/json");
+  callback(resp);
+}