diff options
Diffstat (limited to 'common/common.cpp')
| -rw-r--r-- | common/common.cpp | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/common/common.cpp b/common/common.cpp index 02ec0f8d..6d655fd5 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -801,10 +801,10 @@ std::vector<llama_token> llama_tokenize( // upper limit for the number of tokens int n_tokens = text.length() + add_bos; std::vector<llama_token> result(n_tokens); - n_tokens = llama_tokenize(ctx, text.c_str(), result.data(), result.size(), add_bos); + n_tokens = llama_tokenize(ctx, text.data(), text.length(), result.data(), result.size(), add_bos); if (n_tokens < 0) { result.resize(-n_tokens); - int check = llama_tokenize(ctx, text.c_str(), result.data(), result.size(), add_bos); + int check = llama_tokenize(ctx, text.data(), text.length(), result.data(), result.size(), add_bos); GGML_ASSERT(check == -n_tokens); } else { result.resize(n_tokens); |