diff options
Diffstat (limited to 'convert-hf-to-gguf.py')
| -rwxr-xr-x | convert-hf-to-gguf.py | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 2810e1e4..5ba3161c 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -402,6 +402,7 @@ class Model: # NOTE: this function is generated by convert-hf-to-gguf-update.py # do not modify it manually! # ref: https://github.com/ggerganov/llama.cpp/pull/6920 + # Marker: Start get_vocab_base_pre def get_vocab_base_pre(self, tokenizer) -> str: # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that # is specific for the BPE pre-tokenizer used by the model @@ -489,6 +490,7 @@ class Model: logger.debug(f"chkhsh: {chkhsh}") return res + # Marker: End get_vocab_base_pre def _set_vocab_gpt2(self) -> None: tokens, toktypes, tokpre = self.get_vocab_base() |