fix: dynamically get cuda toolkit version

Author: namchuai

engine/commands/engine_init_cmd.cc

@@ -60,21 +60,22 @@ bool EngineInitCmd::Exec() const {
           variants.push_back(asset_name);
         }
 
-        auto cuda_version = system_info_utils::GetCudaVersion();
-        LOG_INFO << "engineName_: " << engineName_;
-        LOG_INFO << "CUDA version: " << cuda_version;
-        std::string matched_variant = "";
+        auto cuda_driver_version = system_info_utils::GetCudaVersion();
+        LOG_INFO << "Engine: " << engineName_
+                 << ", CUDA driver version: " << cuda_driver_version;
+
+        std::string matched_variant{""};
         if (engineName_ == "cortex.tensorrt-llm") {
           matched_variant = engine_matcher_utils::ValidateTensorrtLlm(
-              variants, system_info.os, cuda_version);
+              variants, system_info.os, cuda_driver_version);
         } else if (engineName_ == "cortex.onnx") {
           matched_variant = engine_matcher_utils::ValidateOnnx(
               variants, system_info.os, system_info.arch);
         } else if (engineName_ == "cortex.llamacpp") {
           auto suitable_avx = engine_matcher_utils::GetSuitableAvxVariant();
           matched_variant = engine_matcher_utils::Validate(
               variants, system_info.os, system_info.arch, suitable_avx,
-              cuda_version);
+              cuda_driver_version);
         }
         LOG_INFO << "Matched variant: " << matched_variant;
         if (matched_variant.empty()) {
@@ -128,22 +129,50 @@ bool EngineInitCmd::Exec() const {
               LOG_INFO << "Finished!";
             });
             if (system_info.os == "mac" || engineName_ == "cortex.onnx") {
-              return false;
+              // mac and onnx engine does not require cuda toolkit
+              return true;
             }
+
             // download cuda toolkit
             const std::string jan_host = "https://catalog.jan.ai";
             const std::string cuda_toolkit_file_name = "cuda.tar.gz";
             const std::string download_id = "cuda";
 
-            auto gpu_driver_version = system_info_utils::GetDriverVersion();
+            // TODO: we don't have API to retrieve list of cuda toolkit dependencies atm
+            // will have better logic after https://github.com/janhq/cortex/issues/1046 finished
+            // for now, assume that we have only 11.7 and 12.4
+            auto suitable_toolkit_version = "";
+            if (engineName_ == "cortex.tensorrt-llm") {
+              // for tensorrt-llm, we need to download cuda toolkit v12.4
+              suitable_toolkit_version = "12.4";
+            } else {
+              // llamacpp
+              if (cuda_driver_version.starts_with("11.")) {
+                suitable_toolkit_version = "11.7";
+              } else if (cuda_driver_version.starts_with("12.")) {
+                suitable_toolkit_version = "12.4";
+              }
+            }
 
-            auto cuda_runtime_version =
-                cuda_toolkit_utils::GetCompatibleCudaToolkitVersion(
-                    gpu_driver_version, system_info.os, engineName_);
+            // compare cuda driver version with cuda toolkit version
+            // cuda driver version should be greater than toolkit version to ensure compatibility
+            if (semantic_version_utils::CompareSemanticVersion(
+                    cuda_driver_version, suitable_toolkit_version) < 0) {
+              LOG_ERROR << "Your Cuda driver version " << cuda_driver_version
+                        << " is not compatible with cuda toolkit version "
+                        << suitable_toolkit_version;
+              return false;
+            }
+
+            std::string cuda_version_path{""};
+            if (!cuda_driver_version.empty()) {
+              cuda_version_path = semantic_version_utils::ConvertToPath(
+                  suitable_toolkit_version);
+            }
 
             std::ostringstream cuda_toolkit_path;
-            cuda_toolkit_path << "dist/cuda-dependencies/" << 11.7 << "/"
-                              << system_info.os << "/"
+            cuda_toolkit_path << "dist/cuda-dependencies/" << cuda_version_path
+                              << "/" << system_info.os << "/"
                               << cuda_toolkit_file_name;
 
             LOG_DEBUG << "Cuda toolkit download url: " << jan_host

engine/utils/engine_matcher_utils.h

@@ -1,5 +1,5 @@
+#include <trantor/utils/Logger.h>
 #include <algorithm>
-#include <iostream>
 #include <iterator>
 #include <regex>
 #include <string>
@@ -177,4 +177,4 @@ inline std::string Validate(const std::vector<std::string>& variants,
 
   return cuda_compatible;
 }
-}  // namespace engine_matcher_utils
+}  // namespace engine_matcher_utils

engine/utils/semantic_version_utils.h

@@ -31,4 +31,14 @@ inline int CompareSemanticVersion(const std::string& version1,
   }
   return 0;
 }
-}  // namespace semantic_version_utils
+
+// convert 11.7 to 11-7 for compatible to download url
+inline std::string ConvertToPath(const std::string& version) {
+  std::string result = version;
+  int pos = result.find('.');
+  if (pos != std::string::npos) {
+    result[pos] = '-';
+  }
+  return result;
+}
+}  // namespace semantic_version_utils