diff options
Diffstat (limited to 'src/llama.cpp')
| -rw-r--r-- | src/llama.cpp | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/src/llama.cpp b/src/llama.cpp index 37653478..54b9b118 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -5552,7 +5552,8 @@ static void llm_load_vocab( } else if ( tokenizer_pre == "llama3" || tokenizer_pre == "llama-v3" || - tokenizer_pre == "llama-bpe") { + tokenizer_pre == "llama-bpe"|| + tokenizer_pre == "falcon3") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3; vocab.tokenizer_ignore_merges = true; vocab.tokenizer_add_bos = true; |