diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/CMakeLists.txt | 3 | ||||
| -rw-r--r-- | tests/test-tokenizer-1.cpp | 16 |
2 files changed, 6 insertions, 13 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4ccefe93..2afaf86b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -28,7 +28,8 @@ llama_build_and_test_executable(test-sampling.cpp) llama_build_executable(test-tokenizer-0.cpp) llama_test_executable (test-tokenizer-0.llama test-tokenizer-0.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf) llama_build_executable(test-tokenizer-1.cpp) -llama_test_executable (test-tokenizer-1.llama test-tokenizer-1.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf) +# test-tokenizer-1 requires a BPE vocab. re-enable when we have one. +#llama_test_executable (test-tokenizer-1.llama test-tokenizer-1.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-falcon.gguf) #llama_test_executable(test-tokenizer-1.aquila test-tokenizer-1.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-aquila.gguf) llama_build_and_test_executable(test-grammar-parser.cpp) llama_build_and_test_executable(test-llama-grammar.cpp) diff --git a/tests/test-tokenizer-1.cpp b/tests/test-tokenizer-1.cpp index 993d17f1..bd607d12 100644 --- a/tests/test-tokenizer-1.cpp +++ b/tests/test-tokenizer-1.cpp @@ -67,11 +67,13 @@ int main(int argc, char **argv) { } } + GGML_ASSERT(llama_vocab_type(ctx) == LLAMA_VOCAB_TYPE_BPE); + const int n_vocab = llama_n_vocab(ctx); for (int i = 0; i < n_vocab; ++i) { - std::string forward = llama_token_to_str_bpe(ctx, i); - std::vector<llama_token> tokens = llama_tokenize_bpe(ctx, forward, false); + std::string forward = llama_token_to_str(ctx, i); + std::vector<llama_token> tokens = llama_tokenize(ctx, forward, false); if (tokens.size() == 1) { if (i != tokens[0]) { std::string backward = llama_token_to_str(ctx, tokens[0]); @@ -79,16 +81,6 @@ int main(int argc, char **argv) { __func__, i, llama_token_to_str(ctx, i).c_str(), tokens[0], backward.c_str()); return 2; } - } else { - llama_token_type type = llama_token_get_type(ctx, i); - if (type == LLAMA_TOKEN_TYPE_UNKNOWN || type == LLAMA_TOKEN_TYPE_CONTROL || type == LLAMA_TOKEN_TYPE_BYTE) { - fprintf(stderr, "%s : info: token %d is string %s and bpe returns tokens %s\n", - __func__, i, llama_token_to_str(ctx, i).c_str(), unescape_whitespace(ctx, tokens).c_str()); - } else { - fprintf(stderr, "%s : error: token %d is string %s but bpe returns tokens %s\n", - __func__, i, llama_token_to_str(ctx, i).c_str(), unescape_whitespace(ctx, tokens).c_str()); - return 2; - } } } |