Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add LoRA support #820

Merged
merged 15 commits into from
Apr 17, 2023
Merged

Add LoRA support #820

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
f52101e
Add lora support
slaren Apr 6, 2023
ac3fbe4
Export lora A matrix pre-transposed
slaren Apr 8, 2023
7136ada
Add support for quantized models
slaren Apr 8, 2023
dc65707
Use the work buffer instead to fix MSVC build
slaren Apr 8, 2023
87c518b
Update exporter and support scaling
slaren Apr 8, 2023
c920f00
Add compatibility with #801
slaren Apr 10, 2023
c45868b
Support more layer types, fix memory and generation issues
slaren Apr 11, 2023
57627f0
Rebase to master
slaren Apr 13, 2023
c150e1b
Add support for using a different base model
slaren Apr 15, 2023
fc89916
Fix windows build
slaren Apr 15, 2023
14858ba
Show warning when using a quantized base model
slaren Apr 15, 2023
3df343b
ggml_cpy: use the work buffer instead of alloca when quantizing
slaren Apr 15, 2023
63da54e
Only attempt to use mmap for the lora base model if it is supported
slaren Apr 16, 2023
0a6d5ad
Reuse definitions from convert.py
slaren Apr 16, 2023
8d37db3
ggml_add: Add more checks
slaren Apr 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Rebase to master
  • Loading branch information
@slaren
slaren committed Apr 16, 2023
commit 57627f0e5faa5bb6b8fd456ccbe1aaf83cff695f
58 changes: 31 additions & 27 deletions ggml.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1420,6 +1420,34 @@ static void dequantize_row_q4_1(const void * restrict vx, float * restrict y, in
#endif
}

static void ggml_vec_dot_q4_1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy);
static void ggml_vec_dot_q4_0_q8_0(const int n, float * restrict s, const void * restrict vx, const void * restrict vy);

static const quantize_fns_t quantize_fns[GGML_TYPE_COUNT] = {
[GGML_TYPE_Q4_0] = {
.dequantize_row_q = dequantize_row_q4_0,
.quantize_row_q = quantize_row_q4_0,
.quantize_row_q_reference = (quantize_row_q_t) quantize_row_q4_0_reference,
.quantize_row_q_dot = quantize_row_q8_0,
.vec_dot_q = ggml_vec_dot_q4_0_q8_0,
},
[GGML_TYPE_Q4_1] = {
.dequantize_row_q = dequantize_row_q4_1,
.quantize_row_q = quantize_row_q4_1,
.quantize_row_q_reference = (quantize_row_q_t) quantize_row_q4_1_reference,
.quantize_row_q_dot = quantize_row_q4_1,
.vec_dot_q = ggml_vec_dot_q4_1,
},
// TODO: GGML_TYPE_Q8_0
};

// For internal test use
quantize_fns_t ggml_internal_get_quantize_fn(size_t i) {
GGML_ASSERT(i < GGML_TYPE_COUNT);
return quantize_fns[i];
}


//
// simd mappings
//
Expand Down Expand Up @@ -5910,12 +5938,12 @@ static void ggml_compute_forward_add_q_f32(
const int64_t ne03 = src0->ne[3];

//const int64_t ne10 = src1->ne[0];
const int64_t ne11 = src1->ne[1];
//const int64_t ne11 = src1->ne[1];
const int64_t ne12 = src1->ne[2];
const int64_t ne13 = src1->ne[3];

const int64_t ne0 = dst->ne[0];
const int64_t ne1 = dst->ne[1];
//const int64_t ne0 = dst->ne[0];
//const int64_t ne1 = dst->ne[1];
const int64_t ne2 = dst->ne[2];
const int64_t ne3 = dst->ne[3];

Expand Down Expand Up @@ -7307,30 +7335,6 @@ static void ggml_compute_forward_mul_mat_f16_f32(
//}
}

static const quantize_fns_t quantize_fns[GGML_TYPE_COUNT] = {
[GGML_TYPE_Q4_0] = {
.dequantize_row_q = dequantize_row_q4_0,
.quantize_row_q = quantize_row_q4_0,
.quantize_row_q_reference = (quantize_row_q_t) quantize_row_q4_0_reference,
.quantize_row_q_dot = quantize_row_q8_0,
.vec_dot_q = ggml_vec_dot_q4_0_q8_0,
},
[GGML_TYPE_Q4_1] = {
.dequantize_row_q = dequantize_row_q4_1,
.quantize_row_q = quantize_row_q4_1,
.quantize_row_q_reference = (quantize_row_q_t) quantize_row_q4_1_reference,
.quantize_row_q_dot = quantize_row_q4_1,
.vec_dot_q = ggml_vec_dot_q4_1,
},
// TODO: GGML_TYPE_Q8_0
};

// For internal test use
quantize_fns_t ggml_internal_get_quantize_fn(size_t i) {
GGML_ASSERT(i < GGML_TYPE_COUNT);
return quantize_fns[i];
}

static void ggml_compute_forward_mul_mat_q_f32(
const struct ggml_compute_params * params,
const struct ggml_tensor * src0,
Expand Down
4 changes: 2 additions & 2 deletions llama.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1896,8 +1896,8 @@ int llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lor
ggml_tensor * loraA = lora_tensors[base_name + ".loraA"];
ggml_tensor * loraB = lora_tensors[base_name + ".loraB"];

if (tensor->ne[0] != loraA->ne[1]) {
fprintf(stderr, "%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");"
if (tensor->ne[0] != loraA->ne[1] || tensor->ne[1] != loraB->ne[1]) {
fprintf(stderr, "%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");"
" are you sure that this adapter is for this model?\n", __func__, tensor->ne[0], loraA->ne[1]);
return 1;
}
Expand Down