From c2624b2fd324ff98cc137397f5b0e1d22869cb58 Mon Sep 17 00:00:00 2001 From: Iwan Kawrakow Date: Thu, 23 Jan 2025 13:10:03 +0200 Subject: Add Deepseek-R1-Distill pre-tokenizer --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/llama.cpp') diff --git a/src/llama.cpp b/src/llama.cpp index d330904f..b983c84b 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -5591,7 +5591,7 @@ static void llm_load_vocab( vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_COMMAND_R; vocab.tokenizer_clean_spaces = false; } else if ( - tokenizer_pre == "qwen2") { + tokenizer_pre == "qwen2" || tokenizer_pre == "deepseek-r1-qwen") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_QWEN2; vocab.tokenizer_clean_spaces = false; } else if ( -- cgit v1.2.3