From 9aa672490c848e45eaa704a554e0f1f6df995fc8 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Date: Mon, 13 May 2024 10:35:14 +0200 Subject: llama : rename jina tokenizers to v2 (#7249) * refactor: rename jina tokenizers to v2 * refactor: keep refactoring non-breaking --- llama.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'llama.cpp') diff --git a/llama.cpp b/llama.cpp index e91ad728..adbcc07e 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4424,7 +4424,9 @@ static void llm_load_vocab( } else if ( tokenizer_pre == "gpt-2" || tokenizer_pre == "jina-es" || - tokenizer_pre == "jina-de") { + tokenizer_pre == "jina-de" || + tokenizer_pre == "jina-v2-es" || + tokenizer_pre == "jina-v2-de") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2; } else if ( tokenizer_pre == "refact") { -- cgit v1.2.3