diff options
| author | hoangmit <hoangmit@users.noreply.github.com> | 2023-03-15 18:41:38 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-03-16 00:41:38 +0200 |
| commit | 6eac39ba953acaeec396cea2969dbf413907e2ec (patch) | |
| tree | 1c263cf2672564d1fba72b2d7e47037cea8345c3 /main.cpp | |
| parent | 27944c4206a49bbe003021a2610bacaa3044e619 (diff) | |
Add RMS norm and use it (#187)
* add ggml_rms_norm
* update op num
Diffstat (limited to 'main.cpp')
| -rw-r--r-- | main.cpp | 6 |
1 files changed, 3 insertions, 3 deletions
@@ -588,7 +588,7 @@ bool llama_eval( // norm { - cur = ggml_norm(ctx0, inpL); + cur = ggml_rms_norm(ctx0, inpL); // cur = attention_norm*cur cur = ggml_mul(ctx0, @@ -678,7 +678,7 @@ bool llama_eval( { // norm { - cur = ggml_norm(ctx0, inpFF); + cur = ggml_rms_norm(ctx0, inpFF); // cur = ffn_norm*cur cur = ggml_mul(ctx0, @@ -713,7 +713,7 @@ bool llama_eval( // norm { - inpL = ggml_norm(ctx0, inpL); + inpL = ggml_rms_norm(ctx0, inpL); // inpL = norm*inpL inpL = ggml_mul(ctx0, |