diff options
| author | Riley Stewart <ristew@users.noreply.github.com> | 2024-02-09 02:49:49 -0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-02-09 12:49:49 +0200 |
| commit | 7c777fcd5dd4af7079e33390cf6a19c328a2666f (patch) | |
| tree | 07df4a7721f19a915c4081625ff8bfaa6134646f /examples/server/server.cpp | |
| parent | e5ca3937c685d6e012ac4db40555d6ec100ff03c (diff) | |
server : fix prompt caching for repeated prompts (#5420)
Diffstat (limited to 'examples/server/server.cpp')
| -rw-r--r-- | examples/server/server.cpp | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/examples/server/server.cpp b/examples/server/server.cpp index eceda30d..8d668f79 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1592,10 +1592,6 @@ struct llama_server_context LOG_TEE("slot %d : in cache: %i tokens | to process: %i tokens\n", slot.id, slot.n_past, slot.num_prompt_tokens_processed); } - LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past); - - llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1); - slot.cache_tokens = prompt_tokens; if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0) @@ -1609,6 +1605,10 @@ struct llama_server_context } } + LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past); + + llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1); + LOG_VERBOSE("prompt ingested", { {"n_past", slot.n_past}, {"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)}, |