From 1a159553f921a9209fed8c714494e57b3649f232 Mon Sep 17 00:00:00 2001 From: staviq Date: Tue, 17 Oct 2023 17:11:01 +0200 Subject: tokenizer : special token handling (#3538) * Rewrite special token handling from #1931 * shorten param name, add st verification by type * use offsets instead of copy by substr * formatting, remove copying iterator on delete * llama : normalize code-style * swift fix * print pfx/sfx if verb, main: split pfx input sfx * dont add space when using special tokens * minor : comment + spacing --------- Co-authored-by: Georgi Gerganov --- common/train.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'common/train.cpp') diff --git a/common/train.cpp b/common/train.cpp index 35a4cf9e..972eaefe 100644 --- a/common/train.cpp +++ b/common/train.cpp @@ -863,7 +863,7 @@ size_t tokenize_file( (int) buf.size(), out_tokens.data(), (int) out_tokens.size(), - false); + false, false); if (n_tokens < 0) { out_tokens.resize(-n_tokens); n_tokens = llama_tokenize( @@ -872,7 +872,7 @@ size_t tokenize_file( (int) buf.size(), out_tokens.data(), (int) out_tokens.size(), - false); + false, false); } if (n_tokens >= 0) { out_tokens.resize(n_tokens); @@ -966,7 +966,7 @@ size_t tokenize_file( (int) buf_sample.size(), tok_sample.data(), (int) tok_sample.size(), - false); + false, false); if (n_tokens < 0) { tok_sample.resize(-n_tokens); n_tokens = llama_tokenize(llama_get_model(lctx), @@ -974,7 +974,7 @@ size_t tokenize_file( (int) buf_sample.size(), tok_sample.data(), (int) tok_sample.size(), - false); + false, false); GGML_ASSERT(n_tokens >= 0); } GGML_ASSERT(n_tokens <= (int) tok_sample.size()); -- cgit v1.2.3