diff options
| author | Kerfuffle <44031344+KerfuffleV2@users.noreply.github.com> | 2023-09-03 04:38:43 -0600 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-09-03 04:38:43 -0600 |
| commit | 6519e9c99cffbad19b31bcba86df48c500628c09 (patch) | |
| tree | 30d405827b056fe55492c5daaf18d915200bad4e | |
| parent | b7f2aa9e512c3be2e863d877cbb1056d7c4a03f8 (diff) | |
gguf(python): Fix special vocab handling when id < 0 (#2984)
| -rw-r--r-- | gguf-py/gguf/gguf.py | 4 | ||||
| -rw-r--r-- | gguf-py/pyproject.toml | 2 |
2 files changed, 3 insertions, 3 deletions
diff --git a/gguf-py/gguf/gguf.py b/gguf-py/gguf/gguf.py index b1bc4205..d377cd56 100644 --- a/gguf-py/gguf/gguf.py +++ b/gguf-py/gguf/gguf.py @@ -801,7 +801,7 @@ class SpecialVocab: else: continue for maybe_token_id in (atok.get('id') for atok in added_tokens if atok.get('content') == tc_content): - if isinstance(maybe_token_id, int): + if isinstance(maybe_token_id, int) and maybe_token_id >= 0: self.special_token_ids[typ] = maybe_token_id break return True @@ -814,7 +814,7 @@ class SpecialVocab: config = json.load(f) for typ in self.special_token_types: maybe_token_id = config.get(f'{typ}_token_id') - if isinstance(maybe_token_id, int): + if isinstance(maybe_token_id, int) and maybe_token_id >= 0: self.special_token_ids[typ] = maybe_token_id return True diff --git a/gguf-py/pyproject.toml b/gguf-py/pyproject.toml index 26f792b1..8da60de1 100644 --- a/gguf-py/pyproject.toml +++ b/gguf-py/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gguf" -version = "0.3.1" +version = "0.3.2" description = "Write ML models in GGUF for GGML" authors = ["GGML <ggml@ggml.ai>"] packages = [ |