diff options
Diffstat (limited to 'examples/server/chat.mjs')
| -rw-r--r-- | examples/server/chat.mjs | 11 |
1 files changed, 11 insertions, 0 deletions
diff --git a/examples/server/chat.mjs b/examples/server/chat.mjs index 87f4d292..219ebb51 100644 --- a/examples/server/chat.mjs +++ b/examples/server/chat.mjs @@ -7,6 +7,11 @@ const args = process.argv.slice(2); const grammarJsonSchemaFile = args.find( (_, index) => args[index - 1] === "--grammar-json-schema" ); + +const no_cached_prompt = args.find( + (_, index) => args[index - 1] === "--no-cache-prompt" +) ?? "false"; + const grammarFile = args.find((_, index) => args[index - 1] === "--grammar"); // Example usage: function,arguments @@ -30,6 +35,9 @@ if (grammarFile) { grammar = readFileSync(grammarFile, 'utf-8') } +// for cached prompt +let slot_id = -1; + const API_URL = 'http://127.0.0.1:8080' const chat = [ @@ -76,6 +84,8 @@ async function chat_completion(question) { top_p: 0.9, n_keep: n_keep, n_predict: 256, + cache_prompt: no_cached_prompt === "false", + slot_id: slot_id, stop: ["\n### Human:"], // stop completion after generating this grammar, stream: true, @@ -92,6 +102,7 @@ async function chat_completion(question) { const t = Buffer.from(chunk).toString('utf8') if (t.startsWith('data: ')) { const message = JSON.parse(t.substring(6)) + slot_id = message.slot_id answer += message.content process.stdout.write(message.content) if (message.stop) { |