diff options
author | Galunid <karolek1231456@gmail.com> | 2023-11-09 11:09:29 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-11-09 11:09:29 +0100 |
commit | a75fa576abba9d37f463580c379e4bbf1e1ad03c (patch) | |
tree | 01b2abda59a4df1382ec42bed67c100aee54a55f /convert.py | |
parent | 57ad015dc3011b046ed5a23186c86ea55f987c54 (diff) |
scripts: Generalize convert scripts (#3838)
* Replace convert-*-hf-to-gguf.py files with convert-hf-to-gguf.py
Diffstat (limited to 'convert.py')
-rwxr-xr-x | convert.py | 4 |
1 files changed, 2 insertions, 2 deletions
@@ -26,7 +26,7 @@ from pathlib import Path from typing import IO, TYPE_CHECKING, Any, Callable, Generator, Iterable, Literal, Sequence, TypeVar import numpy as np -from sentencepiece import SentencePieceProcessor # type: ignore[import] +from sentencepiece import SentencePieceProcessor import os if 'NO_LOCAL_GGUF' not in os.environ: @@ -328,7 +328,7 @@ class BpeVocab: def bpe_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: tokenizer = self.bpe_tokenizer - from transformers.models.gpt2 import tokenization_gpt2 # type: ignore[import] + from transformers.models.gpt2 import tokenization_gpt2 reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.items()} for i, _ in enumerate(tokenizer): |