diff options
Diffstat (limited to 'examples/tokenize/tokenize.cpp')
| -rw-r--r-- | examples/tokenize/tokenize.cpp | 4 |
1 files changed, 1 insertions, 3 deletions
diff --git a/examples/tokenize/tokenize.cpp b/examples/tokenize/tokenize.cpp index d95a9247..8b1baea8 100644 --- a/examples/tokenize/tokenize.cpp +++ b/examples/tokenize/tokenize.cpp @@ -26,11 +26,9 @@ int main(int argc, char ** argv) { llama_context_params ctx_params = llama_context_default_params(); llama_context * ctx = llama_new_context_with_model(model, ctx_params); - const bool add_bos = llama_should_add_bos_token(model); - std::vector<llama_token> tokens; - tokens = ::llama_tokenize(model, prompt, add_bos, true); + tokens = ::llama_tokenize(model, prompt, true, true); for (int i = 0; i < (int) tokens.size(); i++) { if (printing_ids) { |