diff options
Diffstat (limited to 'convert-falcon-hf-to-gguf.py')
-rwxr-xr-x | convert-falcon-hf-to-gguf.py | 23 |
1 files changed, 13 insertions, 10 deletions
diff --git a/convert-falcon-hf-to-gguf.py b/convert-falcon-hf-to-gguf.py index 0fdea70e..ec786ff6 100755 --- a/convert-falcon-hf-to-gguf.py +++ b/convert-falcon-hf-to-gguf.py @@ -1,18 +1,21 @@ #!/usr/bin/env python3 # HF falcon--> gguf conversion -import gguf +from __future__ import annotations + +import argparse +import json import os -import sys import struct -import json +import sys +from pathlib import Path +from typing import Any + +import gguf import numpy as np import torch -import argparse +from transformers import AutoTokenizer # type: ignore[import] -from typing import Any, List -from pathlib import Path -from transformers import AutoTokenizer def bytes_to_unicode(): # ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py @@ -114,9 +117,9 @@ gguf_writer.add_file_type(ftype) print("gguf: get tokenizer metadata") -tokens: List[bytearray] = [] -scores: List[float] = [] -toktypes: List[int] = [] +tokens: list[bytearray] = [] +scores: list[float] = [] +toktypes: list[int] = [] tokenizer_json_file = dir_model / 'tokenizer.json' if not tokenizer_json_file.is_file(): |