Command line switch to use F16 for memory_k and memory_v (refactor of #154) (#294)

* Use F16 for memory_k and memory_v

* add command line switch to use f16 instead of f32 for memory k+v

---------

Co-authored-by: Ty Everett <ty@tyweb.us>

1 files changed, 1 insertions, 0 deletions

diff --git a/utils.h b/utils.h
index e329ba16..c68e4cba 100644
--- a/utils.h
+++ b/utils.h
@@ -18,6 +18,7 @@ struct gpt_params {
     int32_t n_predict = 128; // new tokens to predict
     int32_t repeat_last_n = 64;  // last n tokens to penalize
     int32_t n_ctx = 512; //context size
+    bool memory_f16 = false; // use f16 instead of f32 for memory kv
 
     // sampling parameters
     int32_t top_k = 40;