summaryrefslogtreecommitdiff
path: root/convert.py
diff options
context:
space:
mode:
authorGalunid <karolek1231456@gmail.com>2023-11-09 11:09:29 +0100
committerGitHub <noreply@github.com>2023-11-09 11:09:29 +0100
commita75fa576abba9d37f463580c379e4bbf1e1ad03c (patch)
tree01b2abda59a4df1382ec42bed67c100aee54a55f /convert.py
parent57ad015dc3011b046ed5a23186c86ea55f987c54 (diff)
scripts: Generalize convert scripts (#3838)
* Replace convert-*-hf-to-gguf.py files with convert-hf-to-gguf.py
Diffstat (limited to 'convert.py')
-rwxr-xr-xconvert.py4
1 files changed, 2 insertions, 2 deletions
diff --git a/convert.py b/convert.py
index 9110f158..b0f44dbe 100755
--- a/convert.py
+++ b/convert.py
@@ -26,7 +26,7 @@ from pathlib import Path
from typing import IO, TYPE_CHECKING, Any, Callable, Generator, Iterable, Literal, Sequence, TypeVar
import numpy as np
-from sentencepiece import SentencePieceProcessor # type: ignore[import]
+from sentencepiece import SentencePieceProcessor
import os
if 'NO_LOCAL_GGUF' not in os.environ:
@@ -328,7 +328,7 @@ class BpeVocab:
def bpe_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
tokenizer = self.bpe_tokenizer
- from transformers.models.gpt2 import tokenization_gpt2 # type: ignore[import]
+ from transformers.models.gpt2 import tokenization_gpt2
reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.items()}
for i, _ in enumerate(tokenizer):