From 3e7be3d28e626b52e6198cfc8a20c77a262cfeb9 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Fri, 11 Apr 2025 10:49:18 +0200 Subject: Correct L4 rms_norm (#324) Co-authored-by: Iwan Kawrakow --- src/llama.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/llama.cpp b/src/llama.cpp index 5565ffcd..5340d847 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9913,8 +9913,8 @@ struct llm_build_context { if (model.arch == LLM_ARCH_LLAMA4 && use_rope && hparams.use_kq_norm) { // Llama4TextL2Norm - Qcur = ggml_rms_norm(ctx0, Qcur, 1e-6); - Kcur = ggml_rms_norm(ctx0, Kcur, 1e-6); + Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps); + Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps); cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); } -- cgit v1.2.3