diff options
| author | DAN™ <dranger003@gmail.com> | 2024-05-05 01:19:30 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-05-05 08:19:30 +0300 |
| commit | 889bdd76866ea31a7625ec2dcea63ff469f3e981 (patch) | |
| tree | bcfaea83b66f89181980b64dba57a8c8f8f31a49 /convert-hf-to-gguf.py | |
| parent | 6fbd43221167bf96112f899daf22c127b282cbcf (diff) | |
command-r : add BPE pre-tokenization (#7063)
* Add BPE pre-tokenization for Command-R/R+.
* Bump transformers convert requirement.
* command-r : add individual digits regex
---------
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
Diffstat (limited to 'convert-hf-to-gguf.py')
| -rwxr-xr-x | convert-hf-to-gguf.py | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 88c16676..f7441e6b 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -311,6 +311,9 @@ class Model(ABC): if chkhsh == "6221ad2852e85ce96f791f476e0b390cf9b474c9e3d1362f53a24a06dc8220ff": # ref: https://huggingface.co/smallcloudai/Refact-1_6-base res = "refact" + if chkhsh == "9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8": + # ref: https://huggingface.co/CohereForAI/c4ai-command-r-v01 + res = "command-r" if res is None: logger.warning("\n") |