Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit ba6816f

Browse files
nguyenhoangthuan99nguyenhoangthuan99
authored
feat: models get command (#1035)
1 parent bbc3e31 commit ba6816f

File tree

6 files changed

+231
-1
lines changed

6 files changed

+231
-1
lines changed

‎engine/.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -563,4 +563,5 @@ build
563563
build-deps
564564
.DS_Store
565565

566-
uploads/**
566+
uploads/**
567+
CMakePresets.json

‎engine/commands/model_get_cmd.cc‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#include "model_get_cmd.h"
2+
#include <filesystem>
3+
#include <iostream>
4+
#include <vector>
5+
#include "config/yaml_config.h"
6+
#include "trantor/utils/Logger.h"
7+
#include "utils/cortex_utils.h"
8+
9+
namespace commands {
10+
ModelGetCmd::ModelGetCmd(std::string model_handle)
11+
: model_handle_(std::move(model_handle)) {}
12+
13+
void ModelGetCmd::Exec() {
14+
if (std::filesystem::exists(cortex_utils::models_folder) &&
15+
std::filesystem::is_directory(cortex_utils::models_folder)) {
16+
bool found_model = false;
17+
// Iterate through directory
18+
for (const auto& entry :
19+
std::filesystem::directory_iterator(cortex_utils::models_folder)) {
20+
if (entry.is_regular_file() && entry.path().stem() == model_handle_ &&
21+
entry.path().extension() == ".yaml") {
22+
try {
23+
config::YamlHandler handler;
24+
handler.ModelConfigFromFile(entry.path().string());
25+
const auto& model_config = handler.GetModelConfig();
26+
std::cout << "ModelConfig Details:\n";
27+
std::cout << "-------------------\n";
28+
29+
// Print non-null strings
30+
if (!model_config.id.empty())
31+
std::cout << "id: " << model_config.id << "\n";
32+
if (!model_config.name.empty())
33+
std::cout << "name: " << model_config.name << "\n";
34+
if (!model_config.model.empty())
35+
std::cout << "model: " << model_config.model << "\n";
36+
if (!model_config.version.empty())
37+
std::cout << "version: " << model_config.version << "\n";
38+
39+
// Print non-empty vectors
40+
if (!model_config.stop.empty()) {
41+
std::cout << "stop: [";
42+
for (size_t i = 0; i < model_config.stop.size(); ++i) {
43+
std::cout << model_config.stop[i];
44+
if (i < model_config.stop.size() - 1)
45+
std::cout << ", ";
46+
}
47+
std::cout << "]\n";
48+
}
49+
// Print valid numbers
50+
if (!std::isnan(static_cast<double>(model_config.top_p)))
51+
std::cout << "top_p: " << model_config.top_p << "\n";
52+
if (!std::isnan(static_cast<double>(model_config.temperature)))
53+
std::cout << "temperature: " << model_config.temperature << "\n";
54+
if (!std::isnan(static_cast<double>(model_config.frequency_penalty)))
55+
std::cout << "frequency_penalty: " << model_config.frequency_penalty
56+
<< "\n";
57+
if (!std::isnan(static_cast<double>(model_config.presence_penalty)))
58+
std::cout << "presence_penalty: " << model_config.presence_penalty
59+
<< "\n";
60+
if (!std::isnan(static_cast<double>(model_config.max_tokens)))
61+
std::cout << "max_tokens: " << model_config.max_tokens << "\n";
62+
if (!std::isnan(static_cast<double>(model_config.stream)))
63+
std::cout << "stream: " << std::boolalpha << model_config.stream
64+
<< "\n";
65+
if (!std::isnan(static_cast<double>(model_config.ngl)))
66+
std::cout << "ngl: " << model_config.ngl << "\n";
67+
if (!std::isnan(static_cast<double>(model_config.ctx_len)))
68+
std::cout << "ctx_len: " << model_config.ctx_len << "\n";
69+
70+
// Print non-null strings
71+
if (!model_config.engine.empty())
72+
std::cout << "engine: " << model_config.engine << "\n";
73+
if (!model_config.prompt_template.empty())
74+
std::cout << "prompt_template: " << model_config.prompt_template
75+
<< "\n";
76+
if (!model_config.system_template.empty())
77+
std::cout << "system_template: " << model_config.system_template
78+
<< "\n";
79+
if (!model_config.user_template.empty())
80+
std::cout << "user_template: " << model_config.user_template
81+
<< "\n";
82+
if (!model_config.ai_template.empty())
83+
std::cout << "ai_template: " << model_config.ai_template << "\n";
84+
if (!model_config.os.empty())
85+
std::cout << "os: " << model_config.os << "\n";
86+
if (!model_config.gpu_arch.empty())
87+
std::cout << "gpu_arch: " << model_config.gpu_arch << "\n";
88+
if (!model_config.quantization_method.empty())
89+
std::cout << "quantization_method: "
90+
<< model_config.quantization_method << "\n";
91+
if (!model_config.precision.empty())
92+
std::cout << "precision: " << model_config.precision << "\n";
93+
94+
if (!std::isnan(static_cast<double>(model_config.tp)))
95+
std::cout << "tp: " << model_config.tp << "\n";
96+
97+
// Print non-null strings
98+
if (!model_config.trtllm_version.empty())
99+
std::cout << "trtllm_version: " << model_config.trtllm_version
100+
<< "\n";
101+
if (!std::isnan(static_cast<double>(model_config.text_model)))
102+
std::cout << "text_model: " << std::boolalpha
103+
<< model_config.text_model << "\n";
104+
105+
// Print non-empty vectors
106+
if (!model_config.files.empty()) {
107+
std::cout << "files: [";
108+
for (size_t i = 0; i < model_config.files.size(); ++i) {
109+
std::cout << model_config.files[i];
110+
if (i < model_config.files.size() - 1)
111+
std::cout << ", ";
112+
}
113+
std::cout << "]\n";
114+
}
115+
116+
// Print valid size_t number
117+
if (model_config.created != 0)
118+
std::cout << "created: " << model_config.created << "\n";
119+
120+
if (!model_config.object.empty())
121+
std::cout << "object: " << model_config.object << "\n";
122+
if (!model_config.owned_by.empty())
123+
std::cout << "owned_by: " << model_config.owned_by << "\n";
124+
125+
found_model = true;
126+
break;
127+
} catch (const std::exception& e) {
128+
LOG_ERROR << "Error reading yaml file '" << entry.path().string()
129+
<< "': " << e.what();
130+
}
131+
}
132+
}
133+
}
134+
}
135+
}; // namespace commands

‎engine/commands/model_get_cmd.h‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#pragma once
2+
3+
#include <cmath> // For std::isnan
4+
#include <string>
5+
namespace commands {
6+
7+
class ModelGetCmd {
8+
public:
9+
ModelGetCmd(std::string model_handle);
10+
void Exec();
11+
12+
private:
13+
std::string model_handle_;
14+
};
15+
} // namespace commands

‎engine/controllers/command_line_parser.cc‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#include "command_line_parser.h"
22
#include "commands/engine_init_cmd.h"
33
#include "commands/model_list_cmd.h"
4+
#include "commands/model_get_cmd.h"
5+
46
#include "commands/model_pull_cmd.h"
57
#include "commands/start_model_cmd.h"
68
#include "commands/stop_model_cmd.h"
@@ -51,6 +53,14 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
5153
command.Exec();
5254
});
5355

56+
auto get_models_cmd =
57+
models_cmd->add_subcommand("get", "Get info of {model_id} locally");
58+
get_models_cmd->add_option("model_id", model_id, "");
59+
get_models_cmd->callback([&model_id](){
60+
commands::ModelGetCmd command(model_id);
61+
command.Exec();
62+
});
63+
5464
auto model_pull_cmd =
5565
app_.add_subcommand("pull",
5666
"Download a model from a registry. Working with "

‎engine/controllers/models.cc‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,4 +101,70 @@ void Models::ListModel(
101101
auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
102102
resp->setStatusCode(k200OK);
103103
callback(resp);
104+
}
105+
106+
void Models::GetModel(
107+
const HttpRequestPtr& req,
108+
std::function<void(const HttpResponsePtr&)>&& callback) const {
109+
if (!http_util::HasFieldInReq(req, callback, "modelId")) {
110+
return;
111+
}
112+
auto model_handle = (*(req->getJsonObject())).get("modelId", "").asString();
113+
LOG_DEBUG << "GetModel, Model handle: " << model_handle;
114+
Json::Value ret;
115+
ret["object"] = "list";
116+
Json::Value data(Json::arrayValue);
117+
if (std::filesystem::exists(cortex_utils::models_folder) &&
118+
std::filesystem::is_directory(cortex_utils::models_folder)) {
119+
// Iterate through directory
120+
for (const auto& entry :
121+
std::filesystem::directory_iterator(cortex_utils::models_folder)) {
122+
if (entry.is_regular_file() && entry.path().extension() == ".yaml" &&
123+
entry.path().stem() == model_handle) {
124+
try {
125+
config::YamlHandler handler;
126+
handler.ModelConfigFromFile(entry.path().string());
127+
auto const& model_config = handler.GetModelConfig();
128+
Json::Value obj;
129+
obj["name"] = model_config.name;
130+
obj["model"] = model_config.model;
131+
obj["version"] = model_config.version;
132+
Json::Value stop_array(Json::arrayValue);
133+
for (const std::string& stop : model_config.stop)
134+
stop_array.append(stop);
135+
obj["stop"] = stop_array;
136+
obj["top_p"] = model_config.top_p;
137+
obj["temperature"] = model_config.temperature;
138+
obj["presence_penalty"] = model_config.presence_penalty;
139+
obj["max_tokens"] = model_config.max_tokens;
140+
obj["stream"] = model_config.stream;
141+
obj["ngl"] = model_config.ngl;
142+
obj["ctx_len"] = model_config.ctx_len;
143+
obj["engine"] = model_config.engine;
144+
obj["prompt_template"] = model_config.prompt_template;
145+
146+
Json::Value files_array(Json::arrayValue);
147+
for (const std::string& file : model_config.files)
148+
files_array.append(file);
149+
obj["files"] = files_array;
150+
obj["id"] = model_config.id;
151+
obj["created"] = static_cast<uint32_t>(model_config.created);
152+
obj["object"] = model_config.object;
153+
obj["owned_by"] = model_config.owned_by;
154+
if (model_config.engine == "cortex.tensorrt-llm") {
155+
obj["trtllm_version"] = model_config.trtllm_version;
156+
}
157+
data.append(std::move(obj));
158+
} catch (const std::exception& e) {
159+
LOG_ERROR << "Error reading yaml file '" << entry.path().string()
160+
<< "': " << e.what();
161+
}
162+
}
163+
}
164+
}
165+
ret["data"] = data;
166+
ret["result"] = "OK";
167+
auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret);
168+
resp->setStatusCode(k200OK);
169+
callback(resp);
104170
}

‎engine/controllers/models.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,13 @@ class Models : public drogon::HttpController<Models> {
1414
METHOD_LIST_BEGIN
1515
METHOD_ADD(Models::PullModel, "/pull", Post);
1616
METHOD_ADD(Models::ListModel, "/list", Get);
17+
METHOD_ADD(Models::GetModel, "/get", Post);
1718
METHOD_LIST_END
1819

1920
void PullModel(const HttpRequestPtr& req,
2021
std::function<void(const HttpResponsePtr&)>&& callback) const;
2122
void ListModel(const HttpRequestPtr& req,
2223
std::function<void(const HttpResponsePtr&)>&& callback) const;
24+
void GetModel(const HttpRequestPtr& req,
25+
std::function<void(const HttpResponsePtr&)>&& callback) const;
2326
};

0 commit comments

Comments
 (0)