diff options
| author | Kawrakow <48489457+ikawrakow@users.noreply.github.com> | 2024-08-21 13:00:09 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-08-21 13:00:09 +0300 |
| commit | bd99ed7d0afd2b12c0f5ff5c17b58486396dfe7e (patch) | |
| tree | 2b209a3a58ee6462fedafd538b0fd80f737977ed /src/llama.cpp | |
| parent | d259a50ca6fd3a0821abe6a16b73c0b19c5b4651 (diff) | |
softcap: minor improvement (#24)
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
Diffstat (limited to 'src/llama.cpp')
| -rw-r--r-- | src/llama.cpp | 3 |
1 files changed, 0 insertions, 3 deletions
diff --git a/src/llama.cpp b/src/llama.cpp index 4aee41a4..831f98dc 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -8325,9 +8325,6 @@ static struct ggml_tensor * llm_build_kqv( if (hparams.attn_soft_cap) { kq = ggml_softcap(ctx, kq, 1.0f / hparams.f_attn_logit_softcapping, hparams.f_attn_logit_softcapping); - //kq = ggml_scale(ctx, kq, 1.0f / hparams.f_attn_logit_softcapping); - //kq = ggml_tanh(ctx, kq); - //kq = ggml_scale(ctx, kq, hparams.f_attn_logit_softcapping); } kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias); |