diff options
Diffstat (limited to 'common/common.h')
| -rw-r--r-- | common/common.h | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/common/common.h b/common/common.h index 36fd4416..08c60323 100644 --- a/common/common.h +++ b/common/common.h @@ -137,12 +137,14 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param std::vector<llama_token> llama_tokenize( const struct llama_context * ctx, const std::string & text, - bool add_bos); + bool add_bos, + bool special = false); std::vector<llama_token> llama_tokenize( const struct llama_model * model, const std::string & text, - bool add_bos); + bool add_bos, + bool special = false); // tokenizes a token into a piece // should work similar to Python's `tokenizer.id_to_piece` |