diff options
| author | Anas Ahouzi <112881240+aahouzi@users.noreply.github.com> | 2024-05-19 14:46:46 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-05-19 22:46:46 +1000 |
| commit | 6aade19ee74b896c59929676629340b36be3e22c (patch) | |
| tree | 3c9a5a15ada2e5f87c801a27552598e31d7f8add /convert-hf-to-gguf-update.py | |
| parent | ab33f7a338593f6cf1ae98b10b6f8684f63bd72c (diff) | |
Add StableLM2 pre-tokenizer (#7349)
* Add StableLM pre-tokenizer
* Fix space
* Fix trailing whitespace
Diffstat (limited to 'convert-hf-to-gguf-update.py')
| -rwxr-xr-x | convert-hf-to-gguf-update.py | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index 27983fad..45404b32 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -72,6 +72,7 @@ models = [ {"name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", }, {"name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", }, {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", }, + {"name": "stablelm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b", }, {"name": "refact", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", }, {"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", }, {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", }, |