summaryrefslogtreecommitdiff
path: root/convert-persimmon-to-gguf.py
diff options
context:
space:
mode:
authorGalunid <karolek1231456@gmail.com>2023-11-20 11:35:47 +0100
committerGitHub <noreply@github.com>2023-11-20 11:35:47 +0100
commitf23c0359a32871947169a044eb1dc4dbffd0f405 (patch)
treec427d8031c7d41e1ea23eaeaca956f09b94ee54f /convert-persimmon-to-gguf.py
parent40a34fe8d034bd484efd79ccbb95059ca6308dcb (diff)
ci : add flake8 to github actions (python linting) (#4129)
Disabled rules: * E203 Whitespace before ':' - disabled because we often use 'C' Style where values are aligned * E211 Whitespace before '(' (E211) - disabled because we often use 'C' Style where values are aligned * E221 Multiple spaces before operator - disabled because we often use 'C' Style where values are aligned * E225 Missing whitespace around operator - disabled because it's broken so often it seems like a standard * E231 Missing whitespace after ',', ';', or ':' - disabled because we often use 'C' Style where values are aligned * E241 Multiple spaces after ',' - disabled because we often use 'C' Style where values are aligned * E251 Unexpected spaces around keyword / parameter equals - disabled because it's broken so often it seems like a standard * E261 At least two spaces before inline comment - disabled because it's broken so often it seems like a standard * E266 Too many leading '#' for block comment - sometimes used as "section" separator * E501 Line too long - disabled because it's broken so often it seems like a standard * E701 Multiple statements on one line (colon) - broken only in convert.py when defining abstract methods (we can use# noqa instead) * E704 Multiple statements on one line - broken only in convert.py when defining abstract methods (we can use# noqa instead)
Diffstat (limited to 'convert-persimmon-to-gguf.py')
-rw-r--r--convert-persimmon-to-gguf.py4
1 files changed, 3 insertions, 1 deletions
diff --git a/convert-persimmon-to-gguf.py b/convert-persimmon-to-gguf.py
index 240f8730..206b7d5f 100644
--- a/convert-persimmon-to-gguf.py
+++ b/convert-persimmon-to-gguf.py
@@ -9,6 +9,7 @@ if 'NO_LOCAL_GGUF' not in os.environ:
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
import gguf
+
def _flatten_dict(dct, tensors, prefix=None):
assert isinstance(dct, dict)
for key in dct.keys():
@@ -21,6 +22,7 @@ def _flatten_dict(dct, tensors, prefix=None):
raise ValueError(type(dct[key]))
return None
+
def _get_sentencepiece_tokenizer_info(dir_model: Path):
tokenizer_path = dir_model / 'adept_vocab.model'
print('gguf: getting sentencepiece tokenizer from', tokenizer_path)
@@ -54,6 +56,7 @@ def _get_sentencepiece_tokenizer_info(dir_model: Path):
pass
return tokens, scores, toktypes
+
def main():
parser = argparse.ArgumentParser(description="Convert a Persimmon model from Adept (e.g. Persimmon 8b chat) to a GGML compatible file")
parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input")
@@ -125,6 +128,5 @@ def main():
print("")
-
if __name__ == '__main__':
main()