diff options
author | Cebtenzzre <cebtenzzre@gmail.com> | 2023-09-28 14:30:15 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-09-28 14:30:15 -0400 |
commit | ecf90b1a5114034bc0939b3968f549fe4d63cf6d (patch) | |
tree | edd1cbfc46d4be6484b77e22a27210cfedaf146a /convert-falcon-hf-to-gguf.py | |
parent | 2619109ad57d7a75388a9cce51e5da645410d92e (diff) |
gguf : make token scores and types optional (#3347)
Diffstat (limited to 'convert-falcon-hf-to-gguf.py')
-rwxr-xr-x | convert-falcon-hf-to-gguf.py | 6 |
1 files changed, 0 insertions, 6 deletions
diff --git a/convert-falcon-hf-to-gguf.py b/convert-falcon-hf-to-gguf.py index 88338d82..95835856 100755 --- a/convert-falcon-hf-to-gguf.py +++ b/convert-falcon-hf-to-gguf.py @@ -133,8 +133,6 @@ gguf_writer.add_file_type(ftype) print("gguf: get tokenizer metadata") tokens: list[bytearray] = [] -scores: list[float] = [] -toktypes: list[int] = [] tokenizer_json_file = dir_model / 'tokenizer.json' if not tokenizer_json_file.is_file(): @@ -177,12 +175,8 @@ for i in range(vocab_size): text = bytearray(pad_token) tokens.append(text) - scores.append(0.0) # dymmy - toktypes.append(gguf.TokenType.NORMAL) # dummy gguf_writer.add_token_list(tokens) -gguf_writer.add_token_scores(scores) -gguf_writer.add_token_types(toktypes) special_vocab = gguf.SpecialVocab(dir_model, load_merges = True) special_vocab.add_to_gguf(gguf_writer) |