diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/llama-vocab.cpp | 7 | ||||
| -rw-r--r-- | src/llama.cpp | 4 |
2 files changed, 11 insertions, 0 deletions
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 65ca5e38..474cbd8c 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -477,6 +477,13 @@ struct llm_tokenizer_bpe { "'(?:[sSdDmMtT]|[lL][lL]|[vV][eE]|[rR][eE])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]|\\s+(?!\\S)|\\s+", }; break; + case LLAMA_VOCAB_PRE_TYPE_SEED_CODER: + regex_exprs = { + // original regex from tokenizer.json + // "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1}| ?[^\\s\\p{L}\\p{N}\r\n]+|\\s*[\r\n]+|\\s+(?!\\S)|\\s+" + "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1}| ?[^\\s\\p{L}\\p{N}\\r\\n]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + break; default: // default regex for BPE tokenization pre-processing regex_exprs = { diff --git a/src/llama.cpp b/src/llama.cpp index 3ee95939..564304f6 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -6302,6 +6302,10 @@ static void llm_load_vocab( tokenizer_pre == "bailingmoe") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_BAILINGMOE; vocab.tokenizer_clean_spaces = false; + } else if ( + tokenizer_pre == "seed-coder") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SEED_CODER; + vocab.tokenizer_clean_spaces = false; } else { throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); } |