From 28a2e6e7d476717881be6eb9e2d3331342cec57b Mon Sep 17 00:00:00 2001 From: Kerfuffle <44031344+KerfuffleV2@users.noreply.github.com> Date: Sat, 18 Nov 2023 14:48:17 -0700 Subject: tokenize example: Respect normal add BOS token behavior (#4126) Allow building with Makefile --- examples/tokenize/tokenize.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'examples/tokenize/tokenize.cpp') diff --git a/examples/tokenize/tokenize.cpp b/examples/tokenize/tokenize.cpp index 17166836..4ff8e3fa 100644 --- a/examples/tokenize/tokenize.cpp +++ b/examples/tokenize/tokenize.cpp @@ -26,7 +26,7 @@ int main(int argc, char ** argv) { llama_context_params ctx_params = llama_context_default_params(); llama_context * ctx = llama_new_context_with_model(model, ctx_params); - const bool add_bos = true; + const bool add_bos = llama_should_add_bos_token(model); std::vector tokens; -- cgit v1.2.3