diff options
| author | goerch <jhr.walter@t-online.de> | 2023-10-22 21:21:42 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-10-22 21:21:42 +0200 |
| commit | 9e70cc03229df19ca2d28ce23cc817198f897278 (patch) | |
| tree | 0c027b73d2efc94260b41e2227a1318e2c9ba23d /tests | |
| parent | 5a42a5f8e8a86da9ac88008d748cf232a83aa0e1 (diff) | |
Add test for MPT tokenization (#3728)
* Add test for MPT tokenization
* Revert code motion
* Remove unnecessary restriction in test case
* Clarify logic in conversion
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/CMakeLists.txt | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 61407e57..1c73de0a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -31,6 +31,7 @@ llama_test_executable (test-tokenizer-1-llama test-tokenizer-1-llama.cpp ${CMAKE llama_build_executable(test-tokenizer-1-bpe.cpp) llama_test_executable (test-tokenizer-1-falcon test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-falcon.gguf) llama_test_executable(test-tokenizer-1-aquila test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-aquila.gguf) +llama_test_executable(test-tokenizer-1-mpt test-tokenizer-1-bpe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-mpt.gguf) llama_build_and_test_executable(test-grammar-parser.cpp) llama_build_and_test_executable(test-llama-grammar.cpp) llama_build_and_test_executable(test-grad0.cpp) # SLOW |