diff --git a/convert.py b/convert.py index 14269277627b1..66509b99c8f3e 100644 --- a/convert.py +++ b/convert.py @@ -154,9 +154,15 @@ def guessed(model: 'LazyModel') -> 'Params': # try transformer naming first if "model.layers.0.self_attn.q_proj.weight" in model: n_layer=next(i for i in itertools.count() if f"model.layers.{i}.self_attn.q_proj.weight" not in model) + elif "model.layers.0.self_attn.W_pack.weight" in model: # next: try baichuan naming + n_layer=next(i for i in itertools.count() if f"model.layers.{i}.self_attn.W_pack.weight" not in model) else: n_layer=next(i for i in itertools.count() if f"layers.{i}.attention.wq.weight" not in model) + if n_layer < 1: + raise Exception("failed to guess 'n_layer'. This model is unknown or unsupported.\n" + "Suggestion: provide 'config.json' of the model in the same directory containing model files.") + n_head=n_embd // 128 # guessed return Params( diff --git a/examples/alpaca.sh b/examples/alpaca.sh index aef207f364797..8d2bae6918b62 100755 --- a/examples/alpaca.sh +++ b/examples/alpaca.sh @@ -7,7 +7,7 @@ cd `dirname $0` cd .. -./main -m ./models/ggml-alpaca-7b-q4.bin \ +./main -m ./models/alpaca.13b.ggmlv3.q8_0.bin \ --color \ -f ./prompts/alpaca.txt \ --ctx_size 2048 \ diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index 2b7eb39c51ff5..03e801c2a6d4b 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -18,7 +18,7 @@ int main(int argc, char ** argv) { params.embedding = true; if (params.n_ctx > 2048) { - fprintf(stderr, "%s: warning: model does not support context sizes greater than 2048 tokens (%d specified);" + fprintf(stderr, "%s: warning: model might not support context sizes greater than 2048 tokens (%d specified);" "expect poor results\n", __func__, params.n_ctx); } diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 3a171925ba510..0f6391acba45d 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -85,7 +85,7 @@ int main(int argc, char ** argv) { } if (params.n_ctx > 2048) { - fprintf(stderr, "%s: warning: model does not support context sizes greater than 2048 tokens (%d specified);" + fprintf(stderr, "%s: warning: model might not support context sizes greater than 2048 tokens (%d specified);" "expect poor results\n", __func__, params.n_ctx); } else if (params.n_ctx < 8) { fprintf(stderr, "%s: warning: minimum context size is 8, using minimum size.\n", __func__); diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index dd54ed3c4bd6c..fd4b03cb261f6 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -130,7 +130,7 @@ int main(int argc, char ** argv) { params.n_batch = std::min(params.n_batch, params.n_ctx); if (params.n_ctx > 2048) { - fprintf(stderr, "%s: warning: model does not support context sizes greater than 2048 tokens (%d specified);" + fprintf(stderr, "%s: warning: model might not support context sizes greater than 2048 tokens (%d specified);" "expect poor results\n", __func__, params.n_ctx); } diff --git a/examples/server/README.md b/examples/server/README.md index c5139c16bb827..ad9b6bb081845 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -7,7 +7,7 @@ Command line options: - `--threads N`, `-t N`: Set the number of threads to use during computation. - `-m FNAME`, `--model FNAME`: Specify the path to the LLaMA model file (e.g., `models/7B/ggml-model.bin`). - `-m ALIAS`, `--alias ALIAS`: Set an alias for the model. The alias will be returned in API responses. -- `-c N`, `--ctx-size N`: Set the size of the prompt context. The default is 512, but LLaMA models were built with a context of 2048, which will provide better results for longer input/inference. +- `-c N`, `--ctx-size N`: Set the size of the prompt context. The default is 512, but LLaMA models were built with a context of 2048, which will provide better results for longer input/inference. The size may differ in other models, for example, baichuan models were build with a context of 4096. - `-ngl N`, `--n-gpu-layers N`: When compiled with appropriate support (currently CLBlast or cuBLAS), this option allows offloading some layers to the GPU for computation. Generally results in increased performance. - `-mg i, --main-gpu i`: When using multiple GPUs this option controls which GPU is used for small tensors for which the overhead of splitting the computation across all GPUs is not worthwhile. The GPU in question will use slightly more VRAM to store a scratch buffer for temporary results. By default GPU 0 is used. Requires cuBLAS. - `-ts SPLIT, --tensor-split SPLIT`: When using multiple GPUs this option controls how large tensors should be split across all GPUs. `SPLIT` is a comma-separated list of non-negative values that assigns the proportion of data that each GPU should get in order. For example, "3,2" will assign 60% of the data to GPU 0 and 40% to GPU 1. By default the data is split in proportion to VRAM but this may not be optimal for performance. Requires cuBLAS. diff --git a/ggml-opencl.cpp b/ggml-opencl.cpp index aff33178ed18c..7f642a11f88e9 100644 --- a/ggml-opencl.cpp +++ b/ggml-opencl.cpp @@ -654,13 +654,17 @@ __kernel void dequantize_mul_mat_vec_q6_K(__global const struct block_q6_K * xx, const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... const int in = tid - step*im; // 0...15 or 0...7 -#if K_QUANTS_PER_ITERATION == 1 +\n#if K_QUANTS_PER_ITERATION == 1\n const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 const int is = 0; -#else + +\n#else\n + const int l0 = 4 * in; // 0, 4, 8, ..., 28 const int is = in / 4; -#endif + +\n#endif\n + const int ql_offset = 64*im + l0; const int qh_offset = 32*im + l0; const int s_offset = 8*im + is; @@ -677,7 +681,7 @@ __kernel void dequantize_mul_mat_vec_q6_K(__global const struct block_q6_K * xx, const float d = vload_half(0, &x[i].d); -#if K_QUANTS_PER_ITERATION == 1 +\n#if K_QUANTS_PER_ITERATION == 1\n float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32) + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32) + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32) @@ -687,7 +691,7 @@ __kernel void dequantize_mul_mat_vec_q6_K(__global const struct block_q6_K * xx, + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32) +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32); tmp[16 * ix + tid] += sum; -#else +\n#else\n float sum = 0; for (int l = 0; l < 4; ++l) { sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32) @@ -696,7 +700,7 @@ __kernel void dequantize_mul_mat_vec_q6_K(__global const struct block_q6_K * xx, + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32); } tmp[16 * ix + tid] += sum; -#endif +\n#endif\n } diff --git a/ggml.h b/ggml.h index 0af6dc2804be0..20a8dab856502 100644 --- a/ggml.h +++ b/ggml.h @@ -1516,9 +1516,15 @@ extern "C" { // Internal types and functions exposed for tests and benchmarks // - typedef void (*ggml_to_float_t)(const void * x, float * y, int k); - typedef void (*ggml_from_float_t)(const float * x, void * y, int k); - typedef void (*ggml_vec_dot_t)(const int n, float * s, const void * x, const void * y); +#ifdef __cplusplus +// restrict not standard in C++ +#define GGML_RESTRICT +#else +#define GGML_RESTRICT restrict +#endif + typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int k); + typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int k); + typedef void (*ggml_vec_dot_t) (const int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT x, const void * GGML_RESTRICT y); typedef struct { ggml_to_float_t to_float; diff --git a/koboldcpp.py b/koboldcpp.py index a685df1cc1248..b088b87db5031 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -198,7 +198,6 @@ def load_model(model_filename): inputs.executable_path = (getdirpath()+"/").encode("UTF-8") inputs.debugmode = args.debugmode banned_tokens = args.bantokens - print(banned_tokens) for n in range(ban_token_max): if not banned_tokens or n >= len(banned_tokens): inputs.banned_tokens[n] = "".encode("UTF-8")