From 43248e559472556f368988575d9fba906b3eb139 Mon Sep 17 00:00:00 2001 From: jaime-m-p <167997752+jaime-m-p@users.noreply.github.com> Date: Thu, 9 May 2024 15:30:44 +0200 Subject: llama3 custom regex split (#6965) * merged the changes from deepseeker models to main branch * Moved regex patterns to unicode.cpp and updated unicode.h * Moved header files * Resolved issues * added and refactored unicode_regex_split and related functions * Updated/merged the deepseek coder pr * Refactored code * Adding unicode regex mappings * Adding unicode regex function * Added needed functionality, testing remains * Fixed issues * Fixed issue with gpt2 regex custom preprocessor * unicode : fix? unicode_wstring_to_utf8 * lint : fix whitespaces * tests : add tokenizer tests for numbers * unicode : remove redundant headers * tests : remove and rename tokenizer test scripts * tests : add sample usage * gguf-py : reader prints warnings on duplicate keys * llama : towards llama3 tokenization support (wip) * unicode : shot in the dark to fix tests on Windows * unicode : first try custom implementations * convert : add "tokenizer.ggml.pre" GGUF KV (wip) * llama : use new pre-tokenizer type * convert : fix pre-tokenizer type writing * lint : fix * make : add test-tokenizer-0-llama-v3 * wip * models : add llama v3 vocab file * llama : adapt punctuation regex + add llama 3 regex * minor * unicode : set bomb * unicode : set bomb * unicode : always use std::wregex * unicode : support \p{N}, \p{L} and \p{P} natively * unicode : try fix windows * unicode : category support via std::regex * unicode : clean-up * unicode : simplify * llama3 custom regex split * convert : add convert-hf-to-gguf-update.py ggml-ci * lint : update * convert : add falcon ggml-ci * unicode : normalize signatures * lint : fix * lint : fix * convert : remove unused functions * convert : add comments * convert : exercise contractions ggml-ci * Using char32_t for codepoints * lint : fix * already exists unicode_tolower() * Typing * Restore BOM * cmake : refactor test targets * tests : refactor vocab tests ggml-ci * tests : add more vocabs and tests ggml-ci * unicode : cleanup * scripts : ignore new update script in check-requirements.sh * Fix merge * models : add phi-3, mpt, gpt-2, starcoder * tests : disable obsolete ggml-ci * tests : use faster bpe test ggml-ci * llama : more prominent warning for old BPE models * tests : disable test-tokenizer-1-bpe due to slowness ggml-ci * Move unused variable value * GPT2 custom regex split * Add alternative regex for custom aplit llama3 Co-authored-by: Georgi Gerganov * Style * Add bruteforce random tests for token encoding * wip: fixing unicode codepoint ranges * Fix merge * Unicode tables: separator, lowercase, uppercase and whitespace * llama3 custom regex split: fix \s * Restore BOM * Style * wip: generate NDF table * Ignore special tokens for testing * Clean gen-unicode-data.py * Refactor random tokenizer test * lint : fix * tests : add fail test for llama-bpe --------- Co-authored-by: Jaggzh Co-authored-by: Kazim Abrar Mahi Co-authored-by: Georgi Gerganov Co-authored-by: jaime-m-p <> --- scripts/gen-unicode-data.py | 78 ++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 40 deletions(-) (limited to 'scripts/gen-unicode-data.py') diff --git a/scripts/gen-unicode-data.py b/scripts/gen-unicode-data.py index 977e6561..37d1e396 100644 --- a/scripts/gen-unicode-data.py +++ b/scripts/gen-unicode-data.py @@ -1,31 +1,14 @@ import regex -def cpt_to_utf8_str(cpt): - if cpt <= 0xFF: - return bytes([cpt, 0, 0, 0]) - elif cpt <= 0xFFFF: - return bytes([cpt & 0xFF, cpt >> 8, 0, 0]) - elif cpt <= 0xFFFFFF: - return bytes([cpt & 0xFF, (cpt >> 8) & 0xFF, (cpt >> 16) & 0xFF, 0]) - else: - return bytes([cpt & 0xFF, (cpt >> 8) & 0xFF, (cpt >> 16) & 0xFF, cpt >> 24]) - - -def is_match(codepoint, regex_expr): - try: - res = regex.match(regex_expr, cpt_to_utf8_str(codepoint).decode('utf-32')) - return res is not None - except Exception: - return False - - def get_matches(regex_expr): + regex_expr_compiled = regex.compile(regex_expr) unicode_ranges = [] current_range = None for codepoint in range(0x110000): - if is_match(codepoint, regex_expr): + char = chr(codepoint) + if regex_expr_compiled.match(char): if current_range is None: current_range = [codepoint, codepoint] else: @@ -40,27 +23,42 @@ def get_matches(regex_expr): return unicode_ranges -def print_cat(cat, ranges): - print("const std::vector> unicode_ranges_{} = {{".format(cat)) # noqa: NP100 - cnt = 0 - for start, end in ranges: - if cnt % 4 != 0: - print(" ", end="") # noqa: NP100 - print("{{0x{:08X}, 0x{:08X}}},".format(start, end), end="") # noqa: NP100 - if cnt % 4 == 3: - print("") # noqa: NP100 - cnt += 1 - - if cnt % 4 != 0: - print("") # noqa: NP100 +def print_cat(mode, cat, ranges): + if mode == "range": + print("const std::vector> unicode_ranges_{} = {{".format(cat)) # noqa: NP100 + if mode == "map": + print("const std::map unicode_map_{} = {{".format(cat)) # noqa: NP100 + for i, values in enumerate(ranges): + end = ",\n" if (i % 4 == 3 or i + 1 == len(ranges)) else ", " + values = ["0x%08X" % value for value in values] + print("{" + ", ".join(values) + "}", end=end) # noqa: NP100 print("};") # noqa: NP100 print("") # noqa: NP100 -print_cat("number", get_matches(r'\p{N}')) -print_cat("letter", get_matches(r'\p{L}')) -print_cat("whitespace", get_matches(r'\p{Z}')) -print_cat("accent_mark", get_matches(r'\p{M}')) -print_cat("punctuation", get_matches(r'\p{P}')) -print_cat("symbol", get_matches(r'\p{S}')) -print_cat("control", get_matches(r'\p{C}')) +print_cat("range", "number", get_matches(r'\p{N}')) +print_cat("range", "letter", get_matches(r'\p{L}')) +print_cat("range", "separator", get_matches(r'\p{Z}')) +print_cat("range", "accent_mark", get_matches(r'\p{M}')) +print_cat("range", "punctuation", get_matches(r'\p{P}')) +print_cat("range", "symbol", get_matches(r'\p{S}')) +print_cat("range", "control", get_matches(r'\p{C}')) + +print_cat("range", "whitespace", get_matches(r'\s')) + + +map_lowercase = [] +map_uppercase = [] +for codepoint in range(0x110000): + char = chr(codepoint) + lower = ord(char.lower()[0]) + upper = ord(char.upper()[0]) + if codepoint != lower: + map_lowercase.append((codepoint, lower)) + if codepoint != upper: + map_uppercase.append((codepoint, upper)) +print_cat("map", "lowercase", map_lowercase) +print_cat("map", "uppercase", map_uppercase) + + +# TODO: generate unicode_map_nfd -- cgit v1.2.3