diff options
| author | Ren Xuancheng <jklj077@users.noreply.github.com> | 2024-05-08 20:06:43 +0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-05-08 15:06:43 +0300 |
| commit | 229ffff872f8ad0d21c997d18ee7a23692ae60a0 (patch) | |
| tree | b4c2f055395cc0ccbc2b0ade78da27d429cdb2e1 /convert-hf-to-gguf.py | |
| parent | 1fd9c1741d864d01cd7ec6d67227b92d7bfabf22 (diff) | |
llama : add BPE pre-tokenization for Qwen2 (#7114)
* Add BPE pre-tokenization for Qwen2.
* minor : fixes
---------
Co-authored-by: Ren Xuancheng <17811943+jklj077@users.noreply.github.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
Diffstat (limited to 'convert-hf-to-gguf.py')
| -rwxr-xr-x | convert-hf-to-gguf.py | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 8b89575d..454e9fcd 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -314,6 +314,9 @@ class Model(ABC): if chkhsh == "9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8": # ref: https://huggingface.co/CohereForAI/c4ai-command-r-v01 res = "command-r" + if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea": + # ref: https://huggingface.co/Qwen/Qwen1.5-7B + res = "qwen2" if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166": # ref: https://huggingface.co/allenai/OLMo-1.7-7B-hf res = "olmo" |