diff options
| author | AlpinDale <52078762+AlpinDale@users.noreply.github.com> | 2024-02-23 19:31:54 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-02-23 21:31:54 +0200 |
| commit | fd43d66f46ee3b5345fb8a74a252d86ccd34a409 (patch) | |
| tree | 8d6b2f6446cd74da651da55a20cd6d462123d21e /examples/server/server.cpp | |
| parent | 54fbcd2ce6c48c9e22eca6fbf9e53fb68c3e72ea (diff) | |
server : add KV cache quantization options (#5684)
Diffstat (limited to 'examples/server/server.cpp')
| -rw-r--r-- | examples/server/server.cpp | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 369121e8..524d0ada 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1948,6 +1948,10 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: disabled)\n"); printf(" -spf FNAME, --system-prompt-file FNAME\n"); printf(" set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n"); + printf(" -ctk TYPE, --cache-type-k TYPE\n"); + printf(" KV cache data type for K (default: f16)\n"); + printf(" -ctv TYPE, --cache-type-v TYPE\n"); + printf(" KV cache data type for V (default: f16)\n"); printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); printf(" --log-disable disables logging to a file.\n"); printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n"); @@ -2386,6 +2390,12 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, ); llama.process_system_prompt_data(json::parse(systm_content)); } + else if (arg == "-ctk" || arg == "--cache-type-k") { + params.cache_type_k = argv[++i]; + } + else if (arg == "-ctv" || arg == "--cache-type-v") { + params.cache_type_v = argv[++i]; + } else if(arg == "--mmproj") { if (++i >= argc) |