From bd99ed7d0afd2b12c0f5ff5c17b58486396dfe7e Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Wed, 21 Aug 2024 13:00:09 +0300 Subject: softcap: minor improvement (#24) Co-authored-by: Iwan Kawrakow --- src/llama.cpp | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/llama.cpp') diff --git a/src/llama.cpp b/src/llama.cpp index 4aee41a4..831f98dc 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -8325,9 +8325,6 @@ static struct ggml_tensor * llm_build_kqv( if (hparams.attn_soft_cap) { kq = ggml_softcap(ctx, kq, 1.0f / hparams.f_attn_logit_softcapping, hparams.f_attn_logit_softcapping); - //kq = ggml_scale(ctx, kq, 1.0f / hparams.f_attn_logit_softcapping); - //kq = ggml_tanh(ctx, kq); - //kq = ggml_scale(ctx, kq, hparams.f_attn_logit_softcapping); } kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias); -- cgit v1.2.3