summaryrefslogtreecommitdiff
path: root/convert-hf-to-gguf-update.py
diff options
context:
space:
mode:
Diffstat (limited to 'convert-hf-to-gguf-update.py')
-rw-r--r--convert-hf-to-gguf-update.py275
1 files changed, 275 insertions, 0 deletions
diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py
new file mode 100644
index 00000000..1c559c3f
--- /dev/null
+++ b/convert-hf-to-gguf-update.py
@@ -0,0 +1,275 @@
+# This script downloads the tokenizer models of the specified models from Huggingface and
+# generates the get_vocab_base_pre() function for convert-hf-to-gguf.py
+#
+# This is necessary in order to analyze the type of pre-tokenizer used by the model and
+# provide the necessary information to llama.cpp via the GGUF header in order to implement
+# the same pre-tokenizer.
+#
+# ref: https://github.com/ggerganov/llama.cpp/pull/6920
+#
+# Instructions:
+#
+# - Add a new model to the "models" list
+# - Run the script with your huggingface token:
+#
+# python3 convert-hf-to-gguf-update.py <huggingface_token>
+#
+# - Copy-paste the generated get_vocab_base_pre() function into convert-hf-to-gguf.py
+# - Update llama.cpp with the new pre-tokenizer if necessary
+#
+# TODO: generate tokenizer tests for llama.cpp
+# TODO: automate the update of convert-hf-to-gguf.py
+#
+
+import os
+import requests
+import sys
+import json
+
+from hashlib import sha256
+from enum import IntEnum, auto
+
+class TOKENIZER_TYPE(IntEnum):
+ SPM = auto()
+ BPE = auto()
+ WPM = auto()
+
+# TODO: this string has to exercise as much pre-tokenizer functionality as possible
+# will be updated with time - contributions welcome
+chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天~ ------======= нещо на Български \'\'\'\'\'\'```````\"\"\"\"......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL'
+
+if len(sys.argv) == 2:
+ token = sys.argv[1]
+else:
+ print("Usage: python convert-hf-to-gguf-update.py <huggingface_token>")
+ sys.exit(1)
+
+# TODO: add models here, base models preferred
+models = [
+ { "name": "llama-spm", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", },
+ { "name": "llama-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", },
+ { "name": "phi-3", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", },
+ { "name": "deepseek-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", },
+ { "name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", },
+ { "name": "falcon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", },
+ { "name": "bert-bge", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/BAAI/bge-small-en-v1.5", },
+ { "name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", },
+ { "name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", },
+ { "name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },
+ ]
+
+# make directory "models/tokenizers" if it doesn't exist
+if not os.path.exists("models/tokenizers"):
+ os.makedirs("models/tokenizers")
+
+def download_file_with_auth(url, token, save_path):
+ headers = {"Authorization": f"Bearer {token}"}
+ response = requests.get(url, headers=headers)
+ if response.status_code == 200:
+ with open(save_path, 'wb') as f:
+ f.write(response.content)
+ print(f"File {save_path} downloaded successfully")
+ else:
+ print(f"Failed to download file. Status code: {response.status_code}")
+
+# download the tokenizer models
+for model in models:
+ name = model["name"]
+ repo = model["repo"]
+ tokt = model["tokt"]
+
+ if not os.path.exists(f"models/tokenizers/{name}"):
+ os.makedirs(f"models/tokenizers/{name}")
+ else:
+ print(f"Directory models/tokenizers/{name} already exists - skipping")
+ continue
+
+ print(f"Downloading {name} to models/tokenizers/{name}")
+
+ url = f"{repo}/raw/main/config.json"
+ save_path = f"models/tokenizers/{name}/config.json"
+ download_file_with_auth(url, token, save_path)
+
+ url = f"{repo}/raw/main/tokenizer.json"
+ save_path = f"models/tokenizers/{name}/tokenizer.json"
+ download_file_with_auth(url, token, save_path)
+
+ if tokt == TOKENIZER_TYPE.SPM:
+ url = f"{repo}/resolve/main/tokenizer.model"
+ save_path = f"models/tokenizers/{name}/tokenizer.model"
+ download_file_with_auth(url, token, save_path)
+
+ url = f"{repo}/raw/main/tokenizer_config.json"
+ save_path = f"models/tokenizers/{name}/tokenizer_config.json"
+ download_file_with_auth(url, token, save_path)
+
+# generate the source code for the convert-hf-to-gguf.py:get_vocab_base_pre() function:
+# TODO: auto-update convert-hf-to-gguf.py with the generated function
+
+src_ifs = ""
+for model in models:
+ name = model["name"]
+ tokt = model["tokt"]
+
+ if tokt == TOKENIZER_TYPE.SPM:
+ continue
+
+ # create the tokenizer
+ from transformers import AutoTokenizer
+ tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
+
+ chktok = tokenizer.encode(chktxt)
+ chkhsh = sha256(str(chktok).encode()).hexdigest()
+
+ print(f"model: {name}")
+ print(f"tokt: {tokt}")
+ print(f"repo: {model['repo']}")
+ print(f"chktok: {chktok}")
+ print(f"chkhsh: {chkhsh}")
+
+ # print the "pre_tokenizer" content from the tokenizer.json
+ with open(f"models/tokenizers/{name}/tokenizer.json", "r") as f:
+ cfg = json.load(f)
+ pre_tokenizer = cfg["pre_tokenizer"]
+ print("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))
+
+ print(f"\n")
+
+ src_ifs += f" if chkhsh == \"{chkhsh}\":\n"
+ src_ifs += f" # ref: {model['repo']}\n"
+ src_ifs += f" res = \"{name}\"\n"
+
+src_func = ""
+src_func += " def get_vocab_base_pre(self, tokenizer) -> str:\n"
+src_func += " # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that\n"
+src_func += " # is specific for the BPE pre-tokenizer used by the model\n"
+src_func += " # we will use this unique identifier to write a \"tokenizer.ggml.pre\" entry in the GGUF file which we can\n"
+src_func += " # use in llama.cpp to implement the same pre-tokenizer\n"
+src_func += "\n"
+src_func += f" chktxt = {repr(chktxt)}\n"
+src_func += "\n"
+src_func += " chktok = tokenizer.encode(chktxt)\n"
+src_func += " chkhsh = sha256(str(chktok).encode()).hexdigest()\n"
+src_func += "\n"
+src_func += " print(f\"chktok: {chktok}\")\n"
+src_func += " print(f\"chkhsh: {chkhsh}\")\n"
+src_func += "\n"
+src_func += " res = None\n"
+src_func += "\n"
+src_func += " # NOTE: if you get an error here, you need to add the model to the if-elif chain below\n"
+src_func += " # don't do this manually - use the convert-hf-to-gguf-update.py script!\n"
+src_func += f"{src_ifs}\n"
+src_func += " if res is None:\n"
+src_func += " print(\"\\n\")\n"
+src_func += " print(\"**************************************************************************************\")\n"
+src_func += " print(\"** WARNING: The BPE pre-tokenizer was not recognized!\")\n"
+src_func += " print(\"** This means that it was not added yet or you are using an older version.\")\n"
+src_func += " print(\"** Check convert-hf-to-gguf-update.py and update it accordingly.\")\n"
+src_func += " print(\"**\")\n"
+src_func += " print(f\"** chkhsh: {chkhsh}\")\n"
+src_func += " print(\"**************************************************************************************\")\n"
+src_func += " print(\"\\n\")\n"
+src_func += " raise NotImplementedError(\"BPE pre-tokenizer was not recognized - update get_vocab_base_pre()\")\n"
+src_func += "\n"
+src_func += " print(f\"tokenizer.ggml.pre: {res}\")\n"
+src_func += " print(f\"chkhsh: {chkhsh}\")\n"
+src_func += "\n"
+src_func += " return res\n"
+
+print(src_func)
+
+print("\n")
+print("!!! Copy-paste the function above into convert-hf-to-gguf.py !!!")
+print("\n")
+
+# generate tests for each tokenizer model
+
+tests = [
+ "",
+ " ",
+ " ",
+ " ",
+ "\t",
+ "\n",
+ "\n\n",
+ "\n\n\n",
+ "\t\n",
+ "Hello world",
+ " Hello world",
+ "Hello World",
+ " Hello World",
+ " Hello World!",
+ "Hello, world!",
+ " Hello, world!",
+ " this is 🦙.cpp",
+ "w048 7tuijk dsdfhu",
+ "нещо на Български",
+ "កាន់តែពិសេសអាចខលចេញ",
+ "🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)",
+ "Hello",
+ " Hello",
+ " Hello",
+ " Hello",
+ " Hello",
+ " Hello\n Hello",
+ " (",
+ "\n =",
+ "' era",
+ "Hello, y'all! How are you 😁 ?我想在apple工作1314151天~",
+ "3",
+ "33",
+ "333",
+ "3333",
+ "33333",
+ "333333",
+ "3333333",
+ "33333333",
+ "333333333",
+ chktxt,
+]
+
+# write the tests to ./models/ggml-vocab-{name}.gguf.inp
+# the format is:
+#
+# test0
+# __ggml_vocab_test__
+# test1
+# __ggml_vocab_test__
+# ...
+#
+
+# with each model, encode all tests and write the results in ./models/ggml-vocab-{name}.gguf.out
+# for each test, write the resulting tokens on a separate line
+
+for model in models:
+ name = model["name"]
+ tokt = model["tokt"]
+
+ # create the tokenizer
+ from transformers import AutoTokenizer
+ tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
+
+ with open(f"models/ggml-vocab-{name}.gguf.inp", "w") as f:
+ for text in tests:
+ f.write(f"{text}")
+ f.write("\n__ggml_vocab_test__\n")
+
+ with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f:
+ for text in tests:
+ res = tokenizer.encode(text, add_special_tokens=False)
+ for r in res:
+ f.write(f" {r}")
+ f.write("\n")
+
+ print(f"Tests for {name} written in ./models/ggml-vocab-{name}.gguf.*")
+
+# generate commands for creating vocab files
+
+print("\nRun the following commands to generate the vocab files for testing:\n")
+
+for model in models:
+ name = model["name"]
+
+ print(f"python3 convert-hf-to-gguf.py models/tokenizers/{name}/ --outfile models/ggml-vocab-{name}.gguf --vocab-only")
+
+print("\n")