diff options
| author | Kawrakow <iwankawrakow@gmail.com> | 2025-04-11 10:49:18 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2025-04-11 10:49:18 +0200 |
| commit | 3e7be3d28e626b52e6198cfc8a20c77a262cfeb9 (patch) | |
| tree | 0fccb94db87971fe06aac4b862809a1c998875aa | |
| parent | 474435f58b6a26bc549589966482207fee94aa60 (diff) | |
Correct L4 rms_norm (#324)
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
| -rw-r--r-- | src/llama.cpp | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/src/llama.cpp b/src/llama.cpp index 5565ffcd..5340d847 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9913,8 +9913,8 @@ struct llm_build_context { if (model.arch == LLM_ARCH_LLAMA4 && use_rope && hparams.use_kq_norm) { // Llama4TextL2Norm - Qcur = ggml_rms_norm(ctx0, Qcur, 1e-6); - Kcur = ggml_rms_norm(ctx0, Kcur, 1e-6); + Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps); + Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps); cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); } |