Revert "[model] support vision language model llava. (#178)" (#262)

Let's revert the VLM related change for now to unblock release. cc:
@liutongxuan

- [ ] @guocuimi to create a branch for VLM development.

This reverts commit 437be3f and
d711c55.

Author: guocuimi

python/tests/llava_test.py

@@ -9,7 +9,6 @@ pybind_extension(
     csrc/sampling_params.cpp
     csrc/output.cpp
     csrc/llm_handler.cpp
-    csrc/vlm_handler.cpp
     csrc/module.cpp
   DEPS
     :llm_handler

scalellm/CMakeLists.txt

@@ -2,7 +2,6 @@ from scalellm._C.llm_handler import LLMHandler, Message, Priority
 from scalellm._C.output import (LogProb, LogProbData, RequestOutput,
                                 SequenceOutput, Status, StatusCode, Usage)
 from scalellm._C.sampling_params import SamplingParams
-from scalellm._C.vlm_handler import VLMHandler
 
 # Defined in scalellm/csrc/module.cpp
 def get_metrics() -> str: ...
@@ -19,6 +18,5 @@ __all__ = [
     "StatusCode",
     "Usage",
     "LLMHandler",
-    "VLMHandler",
     "get_metrics",
 ]

scalellm/_C/__init__.pyi

@@ -12,7 +12,7 @@
 
 from scalellm._C import (LLMHandler, LogProb, LogProbData, Message, Priority,
                          RequestOutput, SamplingParams, SequenceOutput, Status,
-                         StatusCode, Usage, VLMHandler, get_metrics)
+                         StatusCode, Usage, get_metrics)
 from scalellm.errors import ValidationError
 from scalellm.llm import LLM
 from scalellm.llm_engine import AsyncLLMEngine, OutputAsyncStream, OutputStream
@@ -34,6 +34,5 @@
     "StatusCode",
     "Usage",
     "LLMHandler",
-    "VLMHandler",
     "get_metrics",
 ]

scalellm/_C/vlm_handler.pyi

@@ -11,7 +11,6 @@ namespace py = pybind11;
 extern void init_sampling_params(py::module_& m);
 extern void init_output(py::module_& m);
 extern void init_llm_handler(py::module_& m);
-extern void init_vlm_handler(py::module_& m);
 
 // NOLINTNEXTLINE
 static std::string get_metrics() { return Metrics::Instance().GetString(); }
@@ -27,7 +26,6 @@ PYBIND11_MODULE(PY_MODULE_NAME, m) {
   init_sampling_params(m);
   init_output(m);
   init_llm_handler(m);
-  init_vlm_handler(m);
 }
 
-}  // namespace llm::csrc
+}  // namespace llm::csrc

scalellm/__init__.py

@@ -10,18 +10,14 @@ cc_library(
     batch.h
     model_runner.h
     worker.h
-    vlm_worker.h
     engine.h
     llm_engine.h
-    vlm_engine.h
   SRCS
     utils.cpp
     batch.cpp
     model_runner.cpp
     worker.cpp
-    vlm_worker.cpp
     llm_engine.cpp
-    vlm_engine.cpp
   DEPS
     torch
     :common

scalellm/csrc/module.cpp

@@ -49,8 +49,6 @@ void Batch::add(Sequence* sequence, uint32_t token_budget) {
   sequences_.push_back(sequence);
   token_budgets_.push_back(token_budget);
   budget_used_.push_back(0);
-
-  input_embedding_ = sequence->get_input_embedding();
 }
 
 void Batch::add(const std::vector<Sequence*>& sequences) {
@@ -260,7 +258,6 @@ ModelInput Batch::prepare_model_input(uint32_t num_decoding_tokens,
 
   pad_2d_vector(block_tables_vec, /*pad_value=*/0);
   input_params.block_tables = create_2d_tensor(block_tables_vec, torch::kInt);
-  input_params.input_embedding = input_embedding_;
 
   CHECK_EQ(sampling_params.size(), selected_token_idxes.size());
   if (!selected_token_idxes.empty()) {