diff options
Diffstat (limited to 'common/common.h')
| -rw-r--r-- | common/common.h | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/common/common.h b/common/common.h index cca44268..157b54a3 100644 --- a/common/common.h +++ b/common/common.h @@ -237,11 +237,12 @@ std::vector<llama_token> llama_tokenize( bool add_special, bool parse_special = false); -// tokenizes a token into a piece +// tokenizes a token into a piece, optionally renders special/control tokens // should work similar to Python's `tokenizer.id_to_piece` std::string llama_token_to_piece( const struct llama_context * ctx, - llama_token token); + llama_token token, + bool special = true); // TODO: these should be moved in llama.h C-style API under single `llama_detokenize` function // that takes into account the tokenizer type and decides how to handle the leading space |