Feat: add multimodal - poc

android/src/main/CMakeLists.txt

@@ -8,6 +8,7 @@ set(RNLLAMA_LIB_DIR ${CMAKE_SOURCE_DIR}/../../../cpp)
 include_directories(
     ${RNLLAMA_LIB_DIR}
     ${RNLLAMA_LIB_DIR}/ggml-cpu
+    ${RNLLAMA_LIB_DIR}/tools/mtmd
 )
 
 set(
@@ -34,6 +35,14 @@ set(
     ${RNLLAMA_LIB_DIR}/gguf.cpp
     ${RNLLAMA_LIB_DIR}/log.cpp
     ${RNLLAMA_LIB_DIR}/llama-impl.cpp
+    # Multimodal support
+    ${RNLLAMA_LIB_DIR}/tools/mtmd/mtmd.cpp
+    ${RNLLAMA_LIB_DIR}/tools/mtmd/mtmd.h
+    ${RNLLAMA_LIB_DIR}/tools/mtmd/clip.cpp
+    ${RNLLAMA_LIB_DIR}/tools/mtmd/clip.h
+    ${RNLLAMA_LIB_DIR}/tools/mtmd/clip-impl.h
+    ${RNLLAMA_LIB_DIR}/tools/mtmd/mtmd-helper.cpp
+    ${RNLLAMA_LIB_DIR}/tools/mtmd/stb_image.h
     ${RNLLAMA_LIB_DIR}/llama-grammar.cpp
     ${RNLLAMA_LIB_DIR}/llama-sampling.cpp
     ${RNLLAMA_LIB_DIR}/llama-vocab.cpp

android/src/main/java/com/rnllama/LlamaContext.java

@@ -309,6 +309,8 @@ public WritableMap completion(ReadableMap params) {
       params.hasKey("top_n_sigma") ? (float) params.getDouble("top_n_sigma") : -1.0f,
       // String[] dry_sequence_breakers, when undef, we use the default definition from common.h
       params.hasKey("dry_sequence_breakers") ? params.getArray("dry_sequence_breakers").toArrayList().toArray(new String[0]) : new String[]{"\n", ":", "\"", "*"},
+      // String[] image_paths
+      params.hasKey("image_paths") ? params.getArray("image_paths").toArrayList().toArray(new String[0]) : new String[0],
       // PartialCompletionCallback partial_completion_callback
       new PartialCompletionCallback(
         this,
@@ -379,6 +381,27 @@ public WritableArray getLoadedLoraAdapters() {
     return getLoadedLoraAdapters(this.context);
   }
 
+  public boolean initMultimodal(ReadableMap params) {
+    String mmprojPath = params.getString("path");
+    boolean mmprojUseGpu = params.hasKey("use_gpu") ? params.getBoolean("use_gpu") : true;
+    if (mmprojPath == null || mmprojPath.isEmpty()) {
+      throw new IllegalArgumentException("mmproj_path is empty");
+    }
+    File file = new File(mmprojPath);
+    if (!file.exists()) {
+      throw new IllegalArgumentException("mmproj file does not exist: " + mmprojPath);
+    }
+    return initMultimodal(this.context, mmprojPath, mmprojUseGpu);
+  }
+
+  public boolean isMultimodalEnabled() {
+    return isMultimodalEnabled(this.context);
+  }
+
+  public void releaseMultimodal() {
+    releaseMultimodal(this.context);
+  }
+
   public void release() {
     freeContext(context);
   }
@@ -497,6 +520,8 @@ protected static native long initContext(
     boolean ctx_shift,
     LoadProgressCallback load_progress_callback
   );
+  protected static native boolean initMultimodal(long contextPtr, String mmproj_path, boolean MMPROJ_USE_GPU);
+  protected static native boolean isMultimodalEnabled(long contextPtr);
   protected static native void interruptLoad(long contextPtr);
   protected static native WritableMap loadModelDetails(
     long contextPtr
@@ -560,6 +585,7 @@ protected static native WritableMap doCompletion(
     int dry_penalty_last_n,
     float top_n_sigma,
     String[] dry_sequence_breakers,
+    String[] image_paths,
     PartialCompletionCallback partial_completion_callback
   );
   protected static native void stopCompletion(long contextPtr);
@@ -579,4 +605,5 @@ protected static native WritableMap embedding(
   protected static native void freeContext(long contextPtr);
   protected static native void setupLog(NativeLogCallback logCallback);
   protected static native void unsetLog();
+  protected static native void releaseMultimodal(long contextPtr);
 }

android/src/main/java/com/rnllama/RNLlama.java

@@ -548,6 +548,106 @@ protected void onPostExecute(ReadableArray result) {
     tasks.put(task, "getLoadedLoraAdapters-" + contextId);
   }
 
+  public void initMultimodal(double id, final ReadableMap params, final Promise promise) {
+    final int contextId = (int) id;
+    AsyncTask task = new AsyncTask<Void, Void, Boolean>() {
+      private Exception exception;
+
+      @Override
+      protected Boolean doInBackground(Void... voids) {
+        try {
+          LlamaContext context = contexts.get(contextId);
+          if (context == null) {
+            throw new Exception("Context not found");
+          }
+          if (context.isPredicting()) {
+            throw new Exception("Context is busy");
+          }
+          return context.initMultimodal(params);
+        } catch (Exception e) {
+          exception = e;
+        }
+        return false;
+      }
+
+      @Override
+      protected void onPostExecute(Boolean result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(result);
+        tasks.remove(this);
+      }
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+    tasks.put(task, "initMultimodal-" + contextId);
+  }
+
+  public void isMultimodalEnabled(double id, final Promise promise) {
+    final int contextId = (int) id;
+    AsyncTask task = new AsyncTask<Void, Void, Boolean>() {
+      private Exception exception;
+
+      @Override
+      protected Boolean doInBackground(Void... voids) {
+        try {
+          LlamaContext context = contexts.get(contextId);
+          if (context == null) {
+            throw new Exception("Context not found");
+          }
+          return context.isMultimodalEnabled();
+        } catch (Exception e) {
+          exception = e;
+        }
+        return false;
+      }
+
+      @Override
+      protected void onPostExecute(Boolean result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(result);
+        tasks.remove(this);
+      }
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+    tasks.put(task, "isMultimodalEnabled" + contextId);
+  }
+
+  @ReactMethod
+  public void releaseMultimodal(double id, final Promise promise) {
+    final int contextId = (int) id;
+    AsyncTask task = new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          LlamaContext context = contexts.get(contextId);
+          if (context == null) {
+            throw new Exception("Context not found");
+          }
+          context.releaseMultimodal();
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+        tasks.remove(this);
+      }
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+    tasks.put(task, "releaseMultimodal" + id);
+  }
+
   public void releaseContext(double id, Promise promise) {
     final int contextId = (int) id;
     AsyncTask task = new AsyncTask<Void, Void, Void>() {

android/src/main/jni.cpp

@@ -685,6 +685,7 @@ Java_com_rnllama_LlamaContext_doCompletion(
     jint dry_penalty_last_n,
     jfloat top_n_sigma,
     jobjectArray dry_sequence_breakers,
+    jobjectArray image_paths,
     jobject partial_completion_callback
 ) {
     UNUSED(thiz);
@@ -694,8 +695,32 @@ Java_com_rnllama_LlamaContext_doCompletion(
 
     //llama_reset_timings(llama->ctx);
 
-    auto prompt_chars = env->GetStringUTFChars(prompt, nullptr);
+    const char *prompt_chars = env->GetStringUTFChars(prompt, nullptr);
+
+    // Set the prompt parameter
     llama->params.prompt = prompt_chars;
+
+    // Process image paths if provided
+    std::vector<std::string> image_paths_vector;
+
+    jint image_paths_size = env->GetArrayLength(image_paths);
+    if (image_paths_size > 0) {
+        // Check if multimodal is enabled
+        if (!llama->isMultimodalEnabled()) {
+            auto result = createWriteableMap(env);
+            putString(env, result, "error", "Multimodal support not enabled. Call initMultimodal first.");
+            env->ReleaseStringUTFChars(prompt, prompt_chars);
+            return reinterpret_cast<jobject>(result);
+        }
+
+        for (jint i = 0; i < image_paths_size; i++) {
+            jstring image_path = (jstring) env->GetObjectArrayElement(image_paths, i);
+            const char *image_path_chars = env->GetStringUTFChars(image_path, nullptr);
+            image_paths_vector.push_back(image_path_chars);
+            env->ReleaseStringUTFChars(image_path, image_path_chars);
+        }
+    }
+
     llama->params.sampling.seed = (seed == -1) ? time(NULL) : seed;
 
     int max_threads = std::thread::hardware_concurrency();
@@ -853,7 +878,7 @@ Java_com_rnllama_LlamaContext_doCompletion(
         return reinterpret_cast<jobject>(result);
     }
     llama->beginCompletion();
-    llama->loadPrompt();
+    llama->loadPrompt(image_paths_vector);
 
     if (llama->context_full) {
         auto result = createWriteableMap(env);
@@ -922,7 +947,12 @@ Java_com_rnllama_LlamaContext_doCompletion(
     }
 
     env->ReleaseStringUTFChars(grammar, grammar_chars);
-    env->ReleaseStringUTFChars(prompt, prompt_chars);
+
+    // Release prompt_chars if it's still allocated
+    if (prompt_chars != nullptr) {
+        env->ReleaseStringUTFChars(prompt, prompt_chars);
+    }
+
     llama_perf_context_print(llama->ctx);
     llama->is_predicting = false;
 
@@ -1098,7 +1128,7 @@ Java_com_rnllama_LlamaContext_embedding(
     }
 
     llama->beginCompletion();
-    llama->loadPrompt();
+    llama->loadPrompt({});
     llama->doCompletion();
 
     std::vector<float> embedding = llama->getEmbedding(embdParams);
@@ -1267,4 +1297,46 @@ Java_com_rnllama_LlamaContext_unsetLog(JNIEnv *env, jobject thiz) {
     llama_log_set(rnllama_log_callback_default, NULL);
 }
 
+JNIEXPORT jboolean JNICALL
+Java_com_rnllama_LlamaContext_initMultimodal(
+    JNIEnv *env,
+    jobject thiz,
+    jlong context_ptr,
+    jstring mmproj_path,
+    jboolean mmproj_use_gpu
+) {
+    UNUSED(thiz);
+    auto llama = context_map[(long) context_ptr];
+
+    const char *mmproj_path_chars = env->GetStringUTFChars(mmproj_path, nullptr);
+    bool result = llama->initMultimodal(mmproj_path_chars, mmproj_use_gpu);
+    env->ReleaseStringUTFChars(mmproj_path, mmproj_path_chars);
+
+    return result;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_com_rnllama_LlamaContext_isMultimodalEnabled(
+    JNIEnv *env,
+    jobject thiz,
+    jlong context_ptr
+) {
+    UNUSED(env);
+    UNUSED(thiz);
+    auto llama = context_map[(long) context_ptr];
+    return llama->isMultimodalEnabled();
+}
+
+JNIEXPORT void JNICALL
+Java_com_rnllama_LlamaContext_releaseMultimodal(
+    JNIEnv *env,
+    jobject thiz,
+    jlong context_ptr
+) {
+    UNUSED(env);
+    UNUSED(thiz);
+    auto llama = context_map[(long) context_ptr];
+    llama->releaseMultimodal();
+}
+
 } // extern "C"

android/src/newarch/java/com/rnllama/RNLlamaModule.java

@@ -52,6 +52,21 @@ public void initContext(double id, final ReadableMap params, final Promise promi
     rnllama.initContext(id, params, promise);
   }
 
+  @ReactMethod
+  public void initMultimodal(double id, final ReadableMap params, final Promise promise) {
+    rnllama.initMultimodal(id, params, promise);
+  }
+
+  @ReactMethod
+  public void isMultimodalEnabled(double id, final Promise promise) {
+    rnllama.isMultimodalEnabled(id, promise);
+  }
+
+  @ReactMethod
+  public void releaseMultimodal(double id, final Promise promise) {
+    rnllama.releaseMultimodal(id, promise);
+  }
+
   @ReactMethod
   public void getFormattedChat(double id, String messages, String chatTemplate, ReadableMap params, Promise promise) {
     rnllama.getFormattedChat(id, messages, chatTemplate, params, promise);

android/src/oldarch/java/com/rnllama/RNLlamaModule.java

@@ -53,6 +53,21 @@ public void initContext(double id, final ReadableMap params, final Promise promi
     rnllama.initContext(id, params, promise);
   }
 
+  @ReactMethod
+  public void initMultimodal(double id, final ReadableMap params, final Promise promise) {
+    rnllama.initMultimodal(id, params, promise);
+  }
+
+  @ReactMethod
+  public void isMultimodalEnabled(double id, final Promise promise) {
+    rnllama.isMultimodalEnabled(id, promise);
+  }
+
+  @ReactMethod
+  public void releaseMultimodal(double id, final Promise promise) {
+    rnllama.releaseMultimodal(id, promise);
+  }
+
   @ReactMethod
   public void getFormattedChat(double id, String messages, String chatTemplate, ReadableMap params, Promise promise) {
     rnllama.getFormattedChat(id, messages, chatTemplate, params, promise);

cpp/chat.cpp

@@ -115,7 +115,9 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
             msgs.push_back(msg);
         }
     } catch (const std::exception & e) {
-        throw std::runtime_error("Failed to parse messages: " + std::string(e.what()) + "; messages = " + messages.dump(2));
+        // @ngxson : disable otherwise it's bloating the API response
+        // printf("%s\n", std::string("; messages = ") + messages.dump(2));
+        throw std::runtime_error("Failed to parse messages: " + std::string(e.what()));
     }
 
     return msgs;

cpp/common.cpp

@@ -1109,7 +1109,6 @@ struct llama_context_params common_context_params_to_llama(const common_params &
     cparams.n_threads         = params.cpuparams.n_threads;
     cparams.n_threads_batch   = params.cpuparams_batch.n_threads == -1 ?
                                 params.cpuparams.n_threads : params.cpuparams_batch.n_threads;
-    cparams.logits_all        = params.logits_all;
     cparams.embeddings        = params.embedding;
     cparams.rope_scaling_type = params.rope_scaling_type;
     cparams.rope_freq_base    = params.rope_freq_base;