From 41b9260f18eb7f325c952006ac46afc1d0d8ad2f Mon Sep 17 00:00:00 2001 From: Elaine Date: Fri, 14 Jun 2024 13:16:49 +0300 Subject: convert : add Poro-34B-chat tokenizer support (#7713) * support for Poro chat pre-tokenizer * add support for Poro pre-tokenizer * Update convert-hf-to-gguf-update.py Co-authored-by: Georgi Gerganov * Change Poro-34B-chat to poro-chat * Change Poro-34B-chat to poro-chat * Update convert-hf-to-gguf-update.py * Update llama.cpp --------- Co-authored-by: Georgi Gerganov --- llama.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'llama.cpp') diff --git a/llama.cpp b/llama.cpp index 225ea977..7f8c259a 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4713,6 +4713,9 @@ static void llm_load_vocab( } else if ( tokenizer_pre == "smaug-bpe") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG; + } else if ( + tokenizer_pre == "poro-chat") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_PORO; } else { throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); } @@ -13028,6 +13031,11 @@ struct llm_tokenizer_bpe { "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", }); break; + case LLAMA_VOCAB_PRE_TYPE_PORO: + word_collection = unicode_regex_split(text, { + " ?[^(\\s|.,!?…。，、।۔،)]+", + }); + break; default: // default regex for BPE tokenization pre-processing word_collection = unicode_regex_split(text, { -- cgit v1.2.3