Skip to content

Commit

Permalink
fixed auto rope scaling (+1 squashed commits)
Browse files Browse the repository at this point in the history 
Squashed commits:

[b1767874] wip
  • Loading branch information
@LostRuins
LostRuins committed Sep 7, 2023
1 parent 0bf75b0 commit 7fb809b
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 28 deletions.
33 changes: 16 additions & 17 deletions expose.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ extern "C"

//return val: 0=fail, 1=(original ggml, alpaca), 2=(ggmf), 3=(ggjt)
static FileFormat file_format = FileFormat::BADFORMAT;
static FileFormatExtraMeta file_format_meta;

bool load_model(const load_model_inputs inputs)
{
Expand All @@ -36,11 +37,9 @@ extern "C"

int forceversion = inputs.forceversion;

if(forceversion==0)
{
file_format = check_file_format(model.c_str());
}
else
file_format = check_file_format(model.c_str(),&file_format_meta);

if(forceversion!=0)
{
printf("\nWARNING: FILE FORMAT FORCED TO VER %d\nIf incorrect, loading may fail or crash.\n",forceversion);
file_format = (FileFormat)forceversion;
Expand All @@ -64,7 +63,7 @@ extern "C"
if(file_format==FileFormat::GPTJ_1 || file_format==FileFormat::GPTJ_2 || file_format==FileFormat::GPTJ_3 || file_format==FileFormat::GPTJ_4 || file_format==FileFormat::GPTJ_5)
{
printf("\n---\nIdentified as GPT-J model: (ver %d)\nAttempting to Load...\n---\n", file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format, file_format_meta);
if (lr == ModelLoadResult::RETRY_LOAD)
{
if(file_format==FileFormat::GPTJ_1)
Expand All @@ -73,22 +72,22 @@ extern "C"
//otherwise if we tried 3 first, then try 2
file_format = FileFormat::GPTJ_4;
printf("\n---\nRetrying as GPT-J model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}

if (lr == ModelLoadResult::RETRY_LOAD)
{
file_format = FileFormat::GPTJ_3;
printf("\n---\nRetrying as GPT-J model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}

//lastly try format 2
if (lr == ModelLoadResult::RETRY_LOAD)
{
file_format = FileFormat::GPTJ_2;
printf("\n---\nRetrying as GPT-J model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
}

Expand All @@ -104,18 +103,18 @@ extern "C"
else if(file_format==FileFormat::GPT2_1||file_format==FileFormat::GPT2_2||file_format==FileFormat::GPT2_3||file_format==FileFormat::GPT2_4)
{
printf("\n---\nIdentified as GPT-2 model: (ver %d)\nAttempting to Load...\n---\n", file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format, file_format_meta);
if (lr == ModelLoadResult::RETRY_LOAD)
{
file_format = FileFormat::GPT2_3;
printf("\n---\nRetrying as GPT-2 model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
if (lr == ModelLoadResult::RETRY_LOAD)
{
file_format = FileFormat::GPT2_2;
printf("\n---\nRetrying as GPT-2 model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
if (lr == ModelLoadResult::FAIL || lr == ModelLoadResult::RETRY_LOAD)
{
Expand All @@ -129,27 +128,27 @@ extern "C"
else if(file_format==FileFormat::NEOX_1 || file_format==FileFormat::NEOX_2 || file_format==FileFormat::NEOX_3 || file_format==FileFormat::NEOX_4 || file_format==FileFormat::NEOX_5 || file_format==FileFormat::NEOX_6 || file_format==FileFormat::NEOX_7)
{
printf("\n---\nIdentified as GPT-NEO-X model: (ver %d)\nAttempting to Load...\n---\n", file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format, file_format_meta);
if (lr == ModelLoadResult::RETRY_LOAD)
{
if(file_format==FileFormat::NEOX_2)
{
file_format = FileFormat::NEOX_3;
printf("\n---\nRetrying as GPT-NEO-X model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
else
{
file_format = FileFormat::NEOX_5;
printf("\n---\nRetrying as GPT-NEO-X model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
}
if (lr == ModelLoadResult::RETRY_LOAD)
{
file_format = FileFormat::NEOX_1;
printf("\n---\nRetrying as GPT-NEO-X model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
if (lr == ModelLoadResult::FAIL || lr == ModelLoadResult::RETRY_LOAD)
{
Expand Down Expand Up @@ -178,7 +177,7 @@ extern "C"
{
printf("\n---\nIdentified as LLAMA model: (ver %d)\nAttempting to Load...\n---\n", file_format);
}
ModelLoadResult lr = gpttype_load_model(inputs, file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format, file_format_meta);
if (lr == ModelLoadResult::FAIL || lr == ModelLoadResult::RETRY_LOAD)
{
return false;
Expand Down
12 changes: 6 additions & 6 deletions gpttype_adapter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ static std::string RemoveBell(const std::string & input) //removes the bell char
return word2;
}

ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format)
ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format, FileFormatExtraMeta file_format_meta)
{
ggml_time_init();

Expand Down Expand Up @@ -438,11 +438,11 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
{
//approximate NTK aware ctx
auto effectivenctx = params.n_ctx;
// if((file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON) && llama_ctx_v4->model.hparams.n_ctx_train>2048)
// {
// float factor = llama_ctx_v4->model.hparams.n_ctx_train/2048;
// effectivenctx = effectivenctx/factor;
// }
if((file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON) && file_format_meta.n_ctx_train > 2048)
{
float factor = file_format_meta.n_ctx_train/2048;
effectivenctx = effectivenctx/factor;
}
rope_freq_base = (effectivenctx <= 3072 ? 26000.0f : (effectivenctx <= 4096 ? 32000.0f : (effectivenctx <= 6144 ? 54000.0f : (effectivenctx <= 8192 ? 82684.0f : (effectivenctx <= 12288 ? 140000.0f : 200000.0f)))));

}
Expand Down
2 changes: 1 addition & 1 deletion koboldcpp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ def utfprint(str):
maxhordelen = 256
modelbusy = threading.Lock()
defaultport = 5001
KcppVersion = "1.42.1"
KcppVersion = "1.43"
showdebug = True
showsamplerwarning = True
showmaxctxwarning = True
Expand Down
14 changes: 12 additions & 2 deletions model_adapter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ void print_tok_vec(std::vector<float> &embd)
}

//return val: 0=fail, 1=(original ggml, alpaca), 2=(ggmf), 3=(ggjt)
FileFormat check_file_format(const std::string & fname)
FileFormat check_file_format(const std::string & fname, FileFormatExtraMeta * fileformatmeta)
{
std::vector<char> f_buf(1024*1024);

Expand Down Expand Up @@ -266,7 +266,7 @@ void print_tok_vec(std::vector<float> &embd)
auto keyidx = gguf_find_key(ctx, "general.architecture");
std::string modelarch = "";
if (keyidx != -1) { modelarch = gguf_get_val_str(ctx, keyidx); }
gguf_free(ctx);

if(modelarch=="llama")
{
fileformat = FileFormat::GGUF_LLAMA;
Expand All @@ -280,6 +280,16 @@ void print_tok_vec(std::vector<float> &embd)
{
printf("\nERROR: Detected unimplemented GGUF Arch: %s\n",modelarch.c_str());
}

if(modelarch!="" && fileformatmeta!=nullptr)
{
std::string fkey = modelarch+".context_length";
auto keyidx = gguf_find_key(ctx, fkey.c_str());
if (keyidx != -1) {
fileformatmeta->n_ctx_train = gguf_get_val_u32(ctx, keyidx);
}
}
gguf_free(ctx);
}

if(fin.is_open())
Expand Down
9 changes: 7 additions & 2 deletions model_adapter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,19 @@ enum FileFormat

};

struct FileFormatExtraMeta
{
int n_ctx_train = 2048;
};

enum ModelLoadResult
{
FAIL = 0,
SUCCESS = 1,
RETRY_LOAD = 2, //used if it's suspected that the model is an older format
};

ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format);
ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format, FileFormatExtraMeta file_format_meta);
generation_outputs gpttype_generate(const generation_inputs inputs, generation_outputs &output);
bool gpttype_generate_abort();
const std::string & gpttype_get_pending_output();
Expand All @@ -73,7 +78,7 @@ std::vector<int> LongestCommonSubseq(const std::vector<int> x, const std::vector
bool ArrStartWith(const std::vector<int> targetArray, const std::vector<int> searchSeq);
int ArrFindIndexOf(const std::vector<int> targetArray, const std::vector<int> searchSeq);

FileFormat check_file_format(const std::string & fname);
FileFormat check_file_format(const std::string & fname, FileFormatExtraMeta * fileformatmeta);
void ContextFastForward(std::vector<int> &current_context_tokens, std::vector<int> &embd_inp,
int &n_past, std::vector<int> &last_n_tokens, const int nctx, std::vector<int> &smartcontext,
const bool useSmartContext, const bool requireFullSubset);

0 comments on commit 7fb809b

Please sign in to comment.