Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ggml-cuda : support stablelm rope #4156

Merged
merged 4 commits into from
Nov 24, 2023
Merged

ggml-cuda : support stablelm rope #4156

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5844493
ggml-cuda : support stablelm rope
slaren Nov 21, 2023
4a3469f
remove unused freq_base kernel parameter
slaren Nov 21, 2023
84adb54
add n_dims parameter to llm_build_k_shift, default to n_rot via overload
slaren Nov 21, 2023
5ed3e1a
llama : fix llm_build_k_shift args
ggerganov Nov 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
llama : fix llm_build_k_shift args
  • Loading branch information
@ggerganov
ggerganov committed Nov 24, 2023
commit 5ed3e1a8f27212bc09c174d11ce5c9c96cc48ec7
24 changes: 4 additions & 20 deletions llama.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3459,8 +3459,7 @@ static void llm_build_k_shift(
struct ggml_cgraph * graph,
llm_rope_type type,
int64_t n_ctx,
int64_t n_rot,
int n_dims,
int n_rot,
float freq_base,
float freq_scale,
const llm_build_cb & cb) {
Expand Down Expand Up @@ -3492,32 +3491,17 @@ static void llm_build_k_shift(
// we rotate only the first n_rot dimensions
ggml_rope_custom_inplace(ctx,
ggml_view_3d(ctx, kv.k,
n_rot, n_head_kv, n_ctx,
n_embd_head, n_head_kv, n_ctx,
ggml_element_size(kv.k)*n_embd_head,
ggml_element_size(kv.k)*n_embd_gqa,
ggml_element_size(kv.k)*n_embd_gqa*n_ctx*il),
K_shift, n_dims, rope_type, 0, n_orig_ctx, freq_base, freq_scale,
K_shift, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
cb(tmp, "K_shifted", il);
ggml_build_forward_expand(graph, tmp);
}
}

static void llm_build_k_shift(
struct ggml_context * ctx,
const llama_hparams & hparams,
const llama_cparams & cparams,
const llama_kv_cache & kv,
struct ggml_cgraph * graph,
llm_rope_type type,
int64_t n_ctx,
int64_t n_rot,
float freq_base,
float freq_scale,
const llm_build_cb & cb) {
llm_build_k_shift(ctx, hparams, cparams, kv, graph, type, n_ctx, n_rot, n_rot, freq_base, freq_scale, cb);
}

static void llm_build_kv_store(
struct ggml_context * ctx,
const llama_hparams & hparams,
Expand Down Expand Up @@ -4814,7 +4798,7 @@ struct llm_build_context {

// shift the entire K-cache if needed
if (do_rope_shift) {
llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, hparams.n_rot, freq_base, freq_scale, cb);
llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, hparams.n_rot, freq_base, freq_scale, cb);
}

for (int il = 0; il < n_layer; ++il) {
Expand Down
Loading