Eval bug: gemma-4-26B-A4B crashing (openweb-ui -> litellm -> llama.cpp version: 8661 (b7ad48ebd)

[mzsergiu]

Name and Version

llama-server --version
version: 8661 (b7ad48e)

Operating systems

Linux

GGML backends

CUDA

Hardware

Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes, VRAM: 24124 MiB
Device 1: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes, VRAM: 24124 MiB
Device 2: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes, VRAM: 24124 MiB
Device 3: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes, VRAM: 24124 MiB
Device 4: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes, VRAM: 24124 MiB
Device 5: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes, VRAM: 24124 MiB

Models

ggml-org/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-Q4_K_M.gguf

Problem description & steps to reproduce

I run:

script to run llama.cpp: gemma-4-26B-A4B-it-q4_k_m.sh
#!/bin/bash
export PATH=/usr/local/cuda-12.8/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/cuda-12.8/lib64:$LD_LIBRARY_PATH
export CUDA_VISIBLE_DEVICES=2

LLAMA_SERVER_BIN="/storage/llm/llama.cpp/build/bin/llama-server"
MODEL_PATH="/storage/llm/models/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-Q4_K_M.gguf"
MMPROJ_PATH="/storage/llm/models/gemma-4-26B-A4B-it-GGUF/mmproj-gemma-4-26B-A4B-it-f16.gguf"

exec "$LLAMA_SERVER_BIN"
-m "$MODEL_PATH"
--mmproj "$MMPROJ_PATH"
--alias gemma-4-26b
--host 0.0.0.0
--port 8001
-np 1
-ngl 99
-fa on
-c 32768
-ctk q8_0
-ctv q8_0
-b 2048
--no-mmap
--no-warmup

It is added in litellm - output from gui

{
"input_cost_per_token": 0,
"output_cost_per_token": 0,
"api_base": "http://127.0.0.1:8001/v1",
"custom_llm_provider": "openai",
"use_in_pass_through": false,
"use_litellm_proxy": false,
"merge_reasoning_content_in_choices": false,
"tags": [],
"model": "gemma-4-26b",
"guardrails": [],
"vector_store_ids": []
}

model used trough litellm in Open WebUI ‧ v0.8.12

First Bad Commit

No response

Relevant log output

Logs

llama.cpp_gemma-4-26B-A4B-it-q4_k_m.txt

[ahmedkhlief]

i also have the same issue where llama.cp crash when i use gemma-4-26B-A4B-it-q4_k_m

[mzsergiu]

To prevent crash of the process:

GGML_ASSERT(src.nx >= 2 && src.ny >= 2); will crash llama.cpp when processing very small images. Fix was implemented to handle 1x1 inputs safely by updating the interpolation math and clamping pixel lookups, preventing out-of-bounds memory errors while keeping the pipeline stable.

Replace resize_bilinear with below code and compile.

File

llama.cpp/tools/mtmd/mtmd-image.cpp

original code

static void resize_bilinear(const clip_image_u8 & src, clip_image_u8 & dst, int target_width, int target_height) {                                                
    GGML_ASSERT(src.nx >= 2 && src.ny >= 2);                                                                                                                      
    dst.nx = target_width;                                                                                                                                        
    dst.ny = target_height;                                                                                                                                       
    dst.buf.resize(3 * target_width * target_height);                                                                                                             
                                                                                                                                                                  
    float x_ratio = static_cast<float>(src.nx - 1) / target_width;                                                                                                
    float y_ratio = static_cast<float>(src.ny - 1) / target_height;                                                                                               
                                                                                                                                                                  
    for (int y = 0; y < target_height; y++) {                                                                                                                     
        for (int x = 0; x < target_width; x++) {                                                                                                                  
            float px = x_ratio * x;                                                                                                                               
            float py = y_ratio * y;                                                                                                                               
            int x_floor = std::min(static_cast<int>(px), src.nx - 2);                                                                                             
            int y_floor = std::min(static_cast<int>(py), src.ny - 2);                                                                                             
            float x_lerp = px - x_floor;                                                                                                                          
            float y_lerp = py - y_floor;                                                                                                                          
                                                                                                                                                                  
            for (int c = 0; c < 3; c++) {                                                                                                                         
                float top = lerp(                                                                                                                                 
                    static_cast<float>(src.buf[3 * (y_floor * src.nx + x_floor) + c]),                                                                            
                    static_cast<float>(src.buf[3 * (y_floor * src.nx + (x_floor + 1)) + c]),                                                                      
                    x_lerp                                                                                                                                        
                );                                                                                                                                                
                float bottom = lerp(                                                                                                                              
                    static_cast<float>(src.buf[3 * ((y_floor + 1) * src.nx + x_floor) + c]),                                                                      
                    static_cast<float>(src.buf[3 * ((y_floor + 1) * src.nx + (x_floor + 1)) + c]),                                                                
                    x_lerp                                                                                                                                        
                );                                                                                                                                                
                dst.buf[3 * (y * target_width + x) + c] = static_cast<uint8_t>(lerp(top, bottom, y_lerp));                                                        
            }                                                                                                                                                     
        }                                                                                                                                                         
    }                                                                                                                                                             
}

updated code

static void resize_bilinear(const clip_image_u8 & src, clip_image_u8 & dst, int target_width, int target_height) {
    if (src.nx == 0 || src.ny == 0) { dst.nx = dst.ny = 0; dst.buf.clear(); return; }
    if (target_width  <= 0) target_width  = 1;
    if (target_height <= 0) target_height = 1;

    dst.nx = target_width;
    dst.ny = target_height;
    dst.buf.resize(3 * target_width * target_height);

    float x_ratio = target_width  > 1 ? static_cast<float>(src.nx - 1) / (target_width  - 1) : 0.0f;
    float y_ratio = target_height > 1 ? static_cast<float>(src.ny - 1) / (target_height - 1) : 0.0f;

    for (int y = 0; y < target_height; ++y) {
        for (int x = 0; x < target_width; ++x) {
            float px = x * x_ratio;
            float py = y * y_ratio;

            int x0 = std::min(static_cast<int>(px), src.nx - 1);
            int y0 = std::min(static_cast<int>(py), src.ny - 1);
            int x1 = std::min(x0 + 1, src.nx - 1);
            int y1 = std::min(y0 + 1, src.ny - 1);

            float xf = px - x0;
            float yf = py - y0;

            for (int c = 0; c < 3; ++c) {
                float top    = lerp(static_cast<float>(src.buf[3 * (y0 * src.nx + x0) + c]),
                                    static_cast<float>(src.buf[3 * (y0 * src.nx + x1) + c]),
                                    xf);
                float bottom = lerp(static_cast<float>(src.buf[3 * (y1 * src.nx + x0) + c]),
                                    static_cast<float>(src.buf[3 * (y1 * src.nx + x1) + c]),
                                    xf);
                dst.buf[3 * (y * target_width + x) + c] = static_cast<uint8_t>(lerp(top, bottom, yf));
            }
        }
    }
}

compile

git pull origin master
rm -rf build
cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 -DBUILD_SHARED_LIBS=OFF -DLLAMA_CURL=ON -DGGML_CUDA_FA_ALL_QUANTS=ON
cmake --build build --config Release -j --clean-first

[Qbason]

I got the same problem with the latest llama.cpp version using Vulkan backend on RX 7900 XTX 24gb. This problem exists in my case for gemma 4 26b and 31b.