Merge branch 'main' into ds-quant

CMakeLists.txt

@@ -169,6 +169,7 @@ set(VLLM_EXT_SRC
   "csrc/quantization/gptq/q_gemm.cu"
   "csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
   "csrc/quantization/fp8/fp8_cuda_kernels.cu"
+  "csrc/quantization/fp8/common.cu"
   "csrc/cuda_utils_kernels.cu"
   "csrc/moe_align_block_size_kernels.cu"
   "csrc/pybind.cpp")

requirements-cuda.txt

@@ -8,4 +8,4 @@ vllm-nccl-cu12>=2.18,<2.19  # for downloading nccl library
 torch == 2.3.0
 xformers == 0.0.26.post1  # Requires PyTorch 2.3.0
 nvidia-cutlass == 3.5.0
-
+vllm-flash-attn == 2.5.8.post1  # Requires PyTorch 2.3.0

vllm/model_executor/layers/quantization/__init__.py

@@ -6,6 +6,8 @@
     QuantizationConfig)
 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import (  # noqa: E501
     CompressedTensorsConfig)
+from vllm.model_executor.layers.quantization.deepspeedfp import (
+    DeepSpeedFPConfig)
 from vllm.model_executor.layers.quantization.fp8 import Fp8Config
 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
 from vllm.model_executor.layers.quantization.gptq_marlin import (
@@ -22,6 +24,7 @@
     "gptq_marlin": GPTQMarlinConfig,
     "marlin": MarlinConfig,
     "sparseml": CompressedTensorsConfig
+    "deepspeedfp": DeepSpeedFPConfig
 }
 
 

vllm/model_executor/models/llama.py

@@ -66,13 +66,13 @@ def __init__(
             layer_name=f"{parent_name}.gate_up_proj",
             input_size=hidden_size,
             output_sizes=[intermediate_size] * 2,
-            bias=False,
+            bias=bias,
             quant_config=quant_config)
         self.down_proj = RowParallelLinear(
             layer_name=f"{parent_name}.down_proj",
             input_size=intermediate_size,
             output_size=hidden_size,
-            bias=False,
+            bias=bias,
             quant_config=quant_config)
         if hidden_act != "silu":
             raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -285,8 +285,10 @@ def __init__(
         self.layers = nn.ModuleList([
             LlamaDecoderLayer(parent_name=f"model.layers.{idx}",
                               config=config,
+                              cache_config=cache_config,
                               quant_config=quant_config)
             for idx in range(config.num_hidden_layers)
+
         ])
         self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)