From ff5a3f0c09dfa0a8e0bf76d1748df5c6dee0e8ff Mon Sep 17 00:00:00 2001 From: goerch Date: Tue, 3 Oct 2023 09:16:26 +0200 Subject: Work on the BPE tokenizer (#3252) * Work on the BPE tokenizer Tokenizer tests work for Falcon-7B * Try to fix build problem * Fix debug assertion failure * Fix MSVC Unicode BOM problem * Cleanup and an improvement * Fix compiler warning * Cleanup * Test doesn't work over the full range of Unicodes * Update .gitignore and Makefile * Another Makefile rule * Testing Aquila * Moving byte decoding back to `token_to_piece` ... ... because everyone is using it. * Guarding some unusable code pathes * Streamlining code and adding some more assertions Important change: I'm classifying added tokens as control tokens now for BPE. * Adding a comment * Adding another assertion * Fixed vocabulary guarding assertions * Fix PR for recent change * Fix PR for recent change * Fix for compiler warning * Fix PR for recent change * Fix PR for recent change * Fix PR for recent change * Fix for compiler warning * Fixes for more compiler warnings * Remove unused code * Fix initialization of static maps * Add scores and token types back, adapt gptneox * Update llama.cpp Co-authored-by: Georgi Gerganov * Update unicode.h Co-authored-by: Georgi Gerganov * Update unicode.h Co-authored-by: Georgi Gerganov * Ported Starcoder and added some assertions * Fix coding style * Apply @jploski 's fix for missing tokens --------- Co-authored-by: Georgi Gerganov --- tests/test-tokenizer-0-falcon.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'tests/test-tokenizer-0-falcon.cpp') diff --git a/tests/test-tokenizer-0-falcon.cpp b/tests/test-tokenizer-0-falcon.cpp index d51851e2..0f3c50bc 100644 --- a/tests/test-tokenizer-0-falcon.cpp +++ b/tests/test-tokenizer-0-falcon.cpp @@ -1,5 +1,6 @@ #include "llama.h" #include "common.h" +#include "console.h" #include #include @@ -85,12 +86,18 @@ int main(int argc, char **argv) { } if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) { - fprintf(stderr, "%s : error: vocab type is not SPM\n", __func__); + fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__); llama_free_model(model); llama_free(ctx); return 2; } +#ifdef _WIN32 + // We need this for unicode console support + console::init(false, false); + atexit([]() { console::cleanup(); }); +#endif + bool success = true; for (const auto & test_kv : k_tests()) { -- cgit v1.2.3