diff options
Diffstat (limited to 'examples/lookup/lookup-stats.cpp')
| -rw-r--r-- | examples/lookup/lookup-stats.cpp | 5 |
1 files changed, 1 insertions, 4 deletions
diff --git a/examples/lookup/lookup-stats.cpp b/examples/lookup/lookup-stats.cpp index 31f22777..41b62c2f 100644 --- a/examples/lookup/lookup-stats.cpp +++ b/examples/lookup/lookup-stats.cpp @@ -34,11 +34,8 @@ int main(int argc, char ** argv){ GGML_ASSERT(llama_n_vocab(model) < (1 << 16)); // tokenize the prompt - const bool add_bos = llama_should_add_bos_token(model); - LOG("add_bos tgt: %d\n", add_bos); - std::vector<llama_token> inp; - inp = ::llama_tokenize(ctx, params.prompt, add_bos, true); + inp = ::llama_tokenize(ctx, params.prompt, true, true); llama_ngram_cache ngram_cache_context; llama_ngram_cache ngram_cache_dynamic; |