feat(openllama): support openllama-3B

application/openllama/README.md

@@ -0,0 +1,44 @@
+## OpenLLaMA-3B
+
+OpenLLaMA 项目地址：https://github.com/openlm-research/open_llama
+
+### 下载 OpenLLaMA-3B 模型
+从 [huggingface](https://huggingface.co/openlm-research/open_llama_3b_600bt_preview/tree/main) 上下载模型，该模型为 fp16 的 pytorch 格式权重
+
+### 量化为 INT4 模型
+量化工具是 cpp 编写的，主要源文件是 quantizer.cpp 文件，运行这个文件之前需要编译固定版本的 llama.cpp。
+```bash
+git clone https://github.com/ggerganov/llama.cpp.git
+cd llama.cpp
+git reset --hard b608b55
+git apply openllama.patch
+mkdir build
+cd build
+cmake ..
+make -j
+cd ..
+python convert.py ${PATH_TO_HUGGINGFACE_OPENLLAMA}/pytorch_model.bin
+./build/bin/quantize ${PATH_TO_HUGGINGFACE_OPENLLAMA}/ggml-model-f16.bin ggml-model-q4_0.bin q4_0
+```
+
+- 克隆仓库后，需要将 commit 回退到 b608b55，因为 InferLLM 最高只支持 ggjt.v1 格式的模型，而 llama.cpp 目前 (commit: 7552ac586380f202b75b18aa216ecfefbd438d94) 已更新到 ggjt.v3 且不向前兼容
+- 回退代码后，需要打上补丁，OpenLLaMa 的 3B 模型的细节配置与 7B 存在不一致，从 pytorch 格式（pytorch_model.bin）转换到 ggjt 格式（ggml-model-f16.bin）时需要特殊处理
+- 编译完成之后在 build 目录下面有一个 bin/quantize 的可执行文件，通过这个工具可以将上一步中的 ggml-model-f16.bin 模型量化为 INT4 的模型（ggml-model-q4_0.bin）
+
+### 运行 OpenLLaMA-3B 模型
+
+可以参考本项目 alpaca 的 README, 编译获得 alpaca 可执行文件。
+```bash
+git clone https://github.com/MegEngine/InferLLM.git
+mkdir build
+cd build
+cmake ..
+make -j
+```
+
+通过 alpaca 可执行文件可以运行量化好的 OpenLLaMA 模型
+
+```bash
+./alpaca -m ggml-model-q4_0.bin -t 4
+```
+

application/openllama/openllama.patch

@@ -0,0 +1,142 @@
+diff --git a/convert.py b/convert.py
+index 8f4f039..ab5047b 100644
+--- a/convert.py
++++ b/convert.py
+@@ -144,12 +144,22 @@ class Params:
+     def guessed(model: 'LazyModel', file_type: GGMLFileType) -> 'Params':
+         n_vocab, n_embd = model["tok_embeddings.weight"].shape
+
++        n_mult = 256
++        n_head = n_embd // 128
++        n_layer = next(i for i in itertools.count() if f"layers.{i}.attention.wq.weight" not in model)
++
++        # TODO: hack for open_llama_3b
++        if n_embd == 3200:
++            n_mult = 216
++            n_head = 32
++            n_layer = 26
++
+         return Params(
+             n_vocab=n_vocab,
+             n_embd=n_embd,
+-            n_mult=256,
+-            n_head=n_embd // 128,
+-            n_layer=next(i for i in itertools.count() if f"layers.{i}.attention.wq.weight" not in model),
++            n_mult=n_mult,
++            n_head=n_head,
++            n_layer=n_layer,
+             file_type=file_type,
+         )
+
+@@ -598,7 +608,9 @@ def convert_transformers_to_orig(model: LazyModel) -> LazyModel:
+     out["norm.weight"] = model["model.norm.weight"]
+     out["output.weight"] = model["lm_head.weight"]
+
+-    n_head = model["model.layers.0.self_attn.q_proj.weight"].shape[1] // 128
++    # TODO: hack for open_llama_3b
++    n_embd = model["model.layers.0.self_attn.q_proj.weight"].shape[1]
++    n_head = 32 if n_embd == 3200 else n_embd // 128
+     for i in itertools.count():
+         if f"model.layers.{i}.self_attn.q_proj.weight" not in model:
+             break
+diff --git a/ggml.c b/ggml.c
+index 4e309df..43947cf 100644
+--- a/ggml.c
++++ b/ggml.c
+@@ -187,6 +187,13 @@ typedef double ggml_float;
+ #include <intrin.h>
+ #else
+ #include <immintrin.h>
++#if (defined(__GNUC__) && __GNUC__ >= 8) || defined(__INTEL_COMPILER)
++#define MM256_SET_M128I(a, b) _mm256_set_m128i((a), (b))
++#define MM256_SET_M128(a, b) _mm256_set_m128((a), (b))
++#else
++#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
++#define MM256_SET_M128(a, b) _mm256_insertf128_ps(_mm256_castps128_ps256(b), (a), 1)
++#endif
+ #endif
+ #endif
+ #endif
+@@ -2985,7 +2992,7 @@ static void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, const void *
+         }
+
+         // Convert int32_t to float
+-        __m256 p = _mm256_cvtepi32_ps( _mm256_set_m128i( i32[0], i32[1] ));
++        __m256 p = _mm256_cvtepi32_ps( MM256_SET_M128I( i32[0], i32[1] ));
+         // Apply the scale, and accumulate
+         acc = _mm256_add_ps(_mm256_mul_ps( d, p ), acc);
+     }
+@@ -3250,11 +3257,11 @@ static void ggml_vec_dot_q4_2_q8_0(const int n, float * restrict s, const void *
+         /* Compute combined scale for the block */
+         const __m128 d0 = _mm_set1_ps(GGML_FP16_TO_FP32(x[2*i + 0].d));
+         const __m128 d1 = _mm_set1_ps(GGML_FP16_TO_FP32(x[2*i + 1].d));
+-        const __m256 d = _mm256_mul_ps(_mm256_set_m128(d1, d0), _mm256_broadcast_ss(&y[i].d));
++        const __m256 d = _mm256_mul_ps(MM256_SET_M128(d1, d0), _mm256_broadcast_ss(&y[i].d));
+
+         __m128i bx0 = bytes_from_nibbles_16(x[2*i + 0].qs);
+         __m128i bx1 = bytes_from_nibbles_16(x[2*i + 1].qs);
+-        __m256i bx = _mm256_set_m128i(bx1, bx0);
++        __m256i bx = MM256_SET_M128I(bx1, bx0);
+
+         // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.
+         const __m256i off = _mm256_set1_epi8(8);
+diff --git a/llama.cpp b/llama.cpp
+index 4bba93a..c3ed784 100644
+--- a/llama.cpp
++++ b/llama.cpp
+@@ -36,6 +36,7 @@
+ // available llama models
+ enum e_model {
+     MODEL_UNKNOWN,
++    MODEL_3B,
+     MODEL_7B,
+     MODEL_13B,
+     MODEL_30B,
+@@ -51,6 +52,7 @@ static const size_t MB = 1024*1024;
+ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH0()
+ {
+     static std::map<e_model, size_t> _MEM_REQ_SCRATCH0 = {
++        { MODEL_3B,    128ull * MB },
+         { MODEL_7B,    512ull * MB },
+         { MODEL_13B,   512ull * MB },
+         { MODEL_30B,   512ull * MB },
+@@ -62,6 +64,7 @@ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH0()
+ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH1()
+ {
+     static std::map<e_model, size_t> _MEM_REQ_SCRATCH1 = {
++        { MODEL_3B,    128ull * MB },
+         { MODEL_7B,    512ull * MB },
+         { MODEL_13B,   512ull * MB },
+         { MODEL_30B,   512ull * MB },
+@@ -74,6 +77,7 @@ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH1()
+ static const std::map<e_model, size_t> & MEM_REQ_KV_SELF()
+ {
+     static std::map<e_model, size_t> _MEM_REQ_KV_SELF = {
++        { MODEL_3B,    682ull * MB },
+         { MODEL_7B,   1026ull * MB },
+         { MODEL_13B,  1608ull * MB },
+         { MODEL_30B,  3124ull * MB },
+@@ -87,6 +91,7 @@ static const std::map<e_model, size_t> & MEM_REQ_KV_SELF()
+ static const std::map<e_model, size_t> & MEM_REQ_EVAL()
+ {
+     static std::map<e_model, size_t> _MEM_REQ_EVAL = {
++        { MODEL_3B,   512ull * MB },
+         { MODEL_7B,   768ull * MB },
+         { MODEL_13B, 1024ull * MB },
+         { MODEL_30B, 1280ull * MB },
+@@ -862,6 +867,7 @@ static const char *llama_ftype_name(enum llama_ftype ftype) {
+
+ static const char *llama_model_type_name(e_model type) {
+     switch (type) {
++        case MODEL_3B: return "3B";
+         case MODEL_7B: return "7B";
+         case MODEL_13B: return "13B";
+         case MODEL_30B: return "30B";
+@@ -894,6 +900,7 @@ static void llama_model_load_internal(
+
+     {
+         switch (hparams.n_layer) {
++            case 26: model.type = e_model::MODEL_3B; break;
+             case 32: model.type = e_model::MODEL_7B; break;
+             case 40: model.type = e_model::MODEL_13B; break;
+             case 60: model.type = e_model::MODEL_30B; break;