diff options
| author | DAN™ <dranger003@gmail.com> | 2024-05-05 01:19:30 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-05-05 08:19:30 +0300 |
| commit | 889bdd76866ea31a7625ec2dcea63ff469f3e981 (patch) | |
| tree | bcfaea83b66f89181980b64dba57a8c8f8f31a49 /tests | |
| parent | 6fbd43221167bf96112f899daf22c127b282cbcf (diff) | |
command-r : add BPE pre-tokenization (#7063)
* Add BPE pre-tokenization for Command-R/R+.
* Bump transformers convert requirement.
* command-r : add individual digits regex
---------
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/CMakeLists.txt | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cad703fc..208f0cf7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -83,6 +83,7 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-bert-bge ARGS ${CMAKE llama_test(test-tokenizer-0 NAME test-tokenizer-0-starcoder ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-starcoder.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-gpt-2 ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-gpt-2.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-refact ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-refact.gguf) +llama_test(test-tokenizer-0 NAME test-tokenizer-0-command-r ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-command-r.gguf) # build test-tokenizer-1-bpe target once and add many tests add_executable(test-tokenizer-1-bpe test-tokenizer-1-bpe.cpp) |