summaryrefslogtreecommitdiff
path: root/tests/test-tokenizer-0-falcon.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'tests/test-tokenizer-0-falcon.cpp')
-rw-r--r--tests/test-tokenizer-0-falcon.cpp9
1 files changed, 8 insertions, 1 deletions
diff --git a/tests/test-tokenizer-0-falcon.cpp b/tests/test-tokenizer-0-falcon.cpp
index d51851e2..0f3c50bc 100644
--- a/tests/test-tokenizer-0-falcon.cpp
+++ b/tests/test-tokenizer-0-falcon.cpp
@@ -1,5 +1,6 @@
#include "llama.h"
#include "common.h"
+#include "console.h"
#include <cstdio>
#include <string>
@@ -85,12 +86,18 @@ int main(int argc, char **argv) {
}
if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) {
- fprintf(stderr, "%s : error: vocab type is not SPM\n", __func__);
+ fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__);
llama_free_model(model);
llama_free(ctx);
return 2;
}
+#ifdef _WIN32
+ // We need this for unicode console support
+ console::init(false, false);
+ atexit([]() { console::cleanup(); });
+#endif
+
bool success = true;
for (const auto & test_kv : k_tests()) {