diff options
| author | DAN™ <dranger003@gmail.com> | 2024-05-08 06:43:23 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-05-08 13:43:23 +0300 |
| commit | 4cd621c26de2095cd7c4464bdec5fe2e696ef3f3 (patch) | |
| tree | 9949775dc9a6ae5c5d1d0370137b1f2bfaf86f62 /convert-hf-to-gguf-update.py | |
| parent | 7e0b6a7b3ba94ff624dc27c1e0e735fded8819b8 (diff) | |
convert : add BPE pre-tokenization for DBRX (#7132)
* Add BPE pre-tokenization for DBRX.
* Add vocab GGUFs.
* Remove test.
* Remove GGUFs.
Diffstat (limited to 'convert-hf-to-gguf-update.py')
| -rwxr-xr-x | convert-hf-to-gguf-update.py | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index a3fe67ee..b5101098 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -68,6 +68,7 @@ models = [ {"name": "refact", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", }, {"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", }, {"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", }, + {"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", }, ] # make directory "models/tokenizers" if it doesn't exist |