diff options
Diffstat (limited to 'convert-hf-to-gguf.py')
| -rwxr-xr-x | convert-hf-to-gguf.py | 93 |
1 files changed, 89 insertions, 4 deletions
diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 3b9fa264..d1b8cef1 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -11,6 +11,7 @@ import sys from abc import ABC, abstractmethod from enum import IntEnum from pathlib import Path +from hashlib import sha256 from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterator, Sequence, TypeVar, cast import numpy as np @@ -229,7 +230,7 @@ class Model(ABC): return (f"pytorch_model-{n:05}-of-{self.num_parts:05}.bin" for n in range(1, self.num_parts + 1)) # used for GPT-2 BPE and WordPiece vocabs - def get_basic_vocab(self) -> tuple[list[str], list[int]]: + def get_vocab_base(self) -> tuple[list[str], list[int], str]: tokens: list[str] = [] toktypes: list[int] = [] @@ -238,6 +239,8 @@ class Model(ABC): vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) assert max(tokenizer.vocab.values()) < vocab_size + tokpre = self.get_vocab_base_pre(tokenizer) + reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} added_vocab = tokenizer.get_added_vocab() @@ -255,11 +258,75 @@ class Model(ABC): tokens.append(reverse_vocab[i]) toktypes.append(gguf.TokenType.NORMAL) - return tokens, toktypes + return tokens, toktypes, tokpre + + # NOTE: this function is generated by convert-hf-to-gguf-update.py + # do not modify it manually! + # ref: https://github.com/ggerganov/llama.cpp/pull/6920 + def get_vocab_base_pre(self, tokenizer) -> str: + # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that + # is specific for the BPE pre-tokenizer used by the model + # we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can + # use in llama.cpp to implement the same pre-tokenizer + + chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български \'\'\'\'\'\'```````""""......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL' + + chktok = tokenizer.encode(chktxt) + chkhsh = sha256(str(chktok).encode()).hexdigest() + + print(f"chktok: {chktok}") + print(f"chkhsh: {chkhsh}") + + res = None + + # NOTE: if you get an error here, you need to add the model to the if-elif chain below + # don't do this manually - use the convert-hf-to-gguf-update.py script! + if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5": + # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B + res = "llama-bpe" + if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754": + # ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base + res = "deepseek-llm" + if chkhsh == "347715f544604f9118bb75ed199f68779f423cabb20db6de6f31b908d04d7821": + # ref: https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base + res = "deepseek-coder" + if chkhsh == "8aeee3860c56296a157a1fe2fad249ec40aa59b1bb5709f4ade11c4e6fe652ed": + # ref: https://huggingface.co/tiiuae/falcon-7b + res = "falcon" + if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f": + # ref: https://huggingface.co/BAAI/bge-small-en-v1.5 + res = "bert-bge" + if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166": + # ref: https://huggingface.co/mosaicml/mpt-7b + res = "mpt" + if chkhsh == "35d91631860c815f952d711435f48d356ebac988362536bed955d43bfa436e34": + # ref: https://huggingface.co/bigcode/starcoder2-3b + res = "starcoder" + if chkhsh == "3ce83efda5659b07b1ad37ca97ca5797ea4285d9b9ab0dc679e4a720c9da7454": + # ref: https://huggingface.co/openai-community/gpt2 + res = "gpt-2" + + if res is None: + print("\n") + print("**************************************************************************************") + print("** WARNING: The BPE pre-tokenizer was not recognized!") + print("** This means that it was not added yet or you are using an older version.") + print("** Check convert-hf-to-gguf-update.py and update it accordingly.") + print("**") + print(f"** chkhsh: {chkhsh}") + print("**************************************************************************************") + print("\n") + raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()") + + print(f"tokenizer.ggml.pre: {res}") + print(f"chkhsh: {chkhsh}") + + return res def _set_vocab_gpt2(self) -> None: - tokens, toktypes = self.get_basic_vocab() + tokens, toktypes, tokpre = self.get_vocab_base() self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) @@ -277,6 +344,8 @@ class Model(ABC): vocab_size = hparams["vocab_size"] assert max(tokenizer.get_vocab().values()) < vocab_size + tokpre = self.get_vocab_base_pre(tokenizer) + merges = [] vocab = {} mergeable_ranks = tokenizer.mergeable_ranks @@ -304,6 +373,7 @@ class Model(ABC): toktypes.append(gguf.TokenType.NORMAL) self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) @@ -376,6 +446,7 @@ class Model(ABC): assert len(tokens) == vocab_size self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_tokenizer_pre("default") self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_scores(scores) self.gguf_writer.add_token_types(toktypes) @@ -397,6 +468,7 @@ class Model(ABC): assert len(tokens) == vocab.vocab_size self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_tokenizer_pre("default") self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_scores(scores) self.gguf_writer.add_token_types(toktypes) @@ -840,6 +912,7 @@ class XverseModel(Model): toktypes.append(toktype) self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_tokenizer_pre("default") self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) @@ -1335,6 +1408,11 @@ class LlamaModel(Model): self.gguf_writer.add_vocab_size(hparams["vocab_size"]) self.gguf_writer.add_rope_dimension_count(hparams["hidden_size"] // hparams["num_attention_heads"]) + if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]: + if self.hparams["rope_scaling"].get("type") == "linear": + self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR) + self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"]) + # Same as super class, but permuting q_proj, k_proj def write_tensors(self): block_count = self.hparams.get("n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer"))) @@ -2052,6 +2130,7 @@ class Phi3MiniModel(Model): toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_tokenizer_pre("default") self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_scores(scores) self.gguf_writer.add_token_types(toktypes) @@ -2294,6 +2373,7 @@ class InternLM2Model(Model): toktypes.append(SentencePieceTokenTypes.USER_DEFINED) self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_tokenizer_pre("default") self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_scores(scores) self.gguf_writer.add_token_types(toktypes) @@ -2443,7 +2523,7 @@ class BertModel(Model): self.gguf_writer.add_pooling_type(pooling_type) def set_vocab(self): - tokens, toktypes = self.get_basic_vocab() + tokens, toktypes, tokpre = self.get_vocab_base() self.vocab_size = len(tokens) # we need this to validate the size of the token_type embeddings @@ -2461,6 +2541,7 @@ class BertModel(Model): # add vocab to gguf self.gguf_writer.add_tokenizer_model("bert") + self.gguf_writer.add_tokenizer_pre(tokpre) self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) @@ -2642,6 +2723,9 @@ class MambaModel(Model): field = neox_reader.get_field(gguf.Keys.Tokenizer.MODEL) self.gguf_writer.add_tokenizer_model(bytes(field.parts[-1])) + field = neox_reader.get_field(gguf.Keys.Tokenizer.PRE) + self.gguf_writer.add_tokenizer_pre(bytes(field.parts[-1])) + field = neox_reader.get_field(gguf.Keys.Tokenizer.LIST) self.gguf_writer.add_token_list([bytes(field.parts[i]) for i in field.data][:vocab_size]) @@ -2847,6 +2931,7 @@ def parse_args() -> argparse.Namespace: help="directory containing model file", ) parser.add_argument("--use-temp-file", action="store_true", help="use the tempfile library while processing (helpful when running out of memory, process killed)") + parser.add_argument("--model-name", type=str, default=None, help="name of the model") return parser.parse_args() |