From eb542d39324574a6778fad9ba9e34ba7a14a82a3 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Tue, 25 Jul 2023 18:35:53 +0300 Subject: Add LLAMA_DEFAULT_RMS_EPS so we can change the default (#2384) Co-authored-by: Iwan Kawrakow --- llama.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'llama.cpp') diff --git a/llama.cpp b/llama.cpp index febefbac..30d4b0a6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -186,7 +186,7 @@ struct llama_hparams { // LLaMAv2 // TODO: load from model data hparams float f_ffn_mult = 1.0f; - float f_rms_norm_eps = 1e-6f; + float f_rms_norm_eps = LLAMA_DEFAULT_RMS_EPS; float rope_freq_base = 10000.0f; float rope_freq_scale = 1.0f; @@ -870,7 +870,7 @@ struct llama_context_params llama_context_default_params() { /*.n_ctx =*/ 512, /*.n_batch =*/ 512, /*.n_gqa =*/ 1, - /*.rms_norm_eps =*/ 1e-6f, + /*.rms_norm_eps =*/ LLAMA_DEFAULT_RMS_EPS, /*.gpu_layers =*/ 0, /*.main_gpu =*/ 0, /*.tensor_split =*/ nullptr, -- cgit v1.2.3