From ecf90b1a5114034bc0939b3968f549fe4d63cf6d Mon Sep 17 00:00:00 2001 From: Cebtenzzre Date: Thu, 28 Sep 2023 14:30:15 -0400 Subject: gguf : make token scores and types optional (#3347) --- convert-starcoder-hf-to-gguf.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'convert-starcoder-hf-to-gguf.py') diff --git a/convert-starcoder-hf-to-gguf.py b/convert-starcoder-hf-to-gguf.py index 331e84e9..48e88a77 100755 --- a/convert-starcoder-hf-to-gguf.py +++ b/convert-starcoder-hf-to-gguf.py @@ -117,8 +117,6 @@ gguf_writer.add_file_type(ftype) print("gguf: get tokenizer metadata") tokens: list[bytearray] = [] -scores: list[float] = [] -toktypes: list[int] = [] tokenizer_json_file = dir_model / 'tokenizer.json' if not tokenizer_json_file.is_file(): @@ -161,12 +159,8 @@ for i in range(vocab_size): text = bytearray(pad_token) tokens.append(text) - scores.append(0.0) # dymmy - toktypes.append(gguf.TokenType.NORMAL) # dummy gguf_writer.add_token_list(tokens) -gguf_writer.add_token_scores(scores) -gguf_writer.add_token_types(toktypes) special_vocab = gguf.SpecialVocab(dir_model, load_merges = True) special_vocab.add_to_gguf(gguf_writer) -- cgit v1.2.3