From 0b366e735729327476ec31da02de3c9c9771ddfb Mon Sep 17 00:00:00 2001 From: Erik Scholz Date: Sun, 19 Mar 2023 18:57:00 +0100 Subject: Command line switch to use F16 for memory_k and memory_v (refactor of #154) (#294) * Use F16 for memory_k and memory_v * add command line switch to use f16 instead of f32 for memory k+v --------- Co-authored-by: Ty Everett --- utils.h | 1 + 1 file changed, 1 insertion(+) (limited to 'utils.h') diff --git a/utils.h b/utils.h index e329ba16..c68e4cba 100644 --- a/utils.h +++ b/utils.h @@ -18,6 +18,7 @@ struct gpt_params { int32_t n_predict = 128; // new tokens to predict int32_t repeat_last_n = 64; // last n tokens to penalize int32_t n_ctx = 512; //context size + bool memory_f16 = false; // use f16 instead of f32 for memory kv // sampling parameters int32_t top_k = 40; -- cgit v1.2.3