diff options
| author | Johannes Gäßler <johannesg@5d6.de> | 2024-05-17 09:59:57 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-05-17 09:59:57 +0200 |
| commit | 29c60d8cddcfd14fa8a6bf023a6c4eb8692c76ba (patch) | |
| tree | 17eb4230f90fa521ecd9809c8b36d3c9276a2eee /llama.cpp | |
| parent | 359cbe3f46c90ce6f5151005e411b8fb74f8139e (diff) | |
tokenization: add warning for double BOS (#7332)
Diffstat (limited to 'llama.cpp')
| -rw-r--r-- | llama.cpp | 14 |
1 files changed, 14 insertions, 0 deletions
@@ -12818,6 +12818,13 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab & } } + if (add_special && vocab.special_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) { + LLAMA_LOG_WARN( + "%s: Added a BOS token to the prompt as specified by the model but the prompt " + "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. " + "Are you sure this is what you want?\n", __FUNCTION__); + } + if (add_special && vocab.special_add_eos == 1) { GGML_ASSERT(vocab.special_eos_id != -1); output.push_back(vocab.special_eos_id); @@ -12844,6 +12851,13 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab & } } + if (add_special && vocab.special_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) { + LLAMA_LOG_WARN( + "%s: Added a BOS token to the prompt as specified by the model but the prompt " + "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. " + "Are you sure this is what you want?\n", __FUNCTION__); + } + if (add_special && vocab.special_add_eos == 1) { GGML_ASSERT(vocab.special_add_eos != -1); output.push_back(vocab.special_eos_id); |