From 6519e9c99cffbad19b31bcba86df48c500628c09 Mon Sep 17 00:00:00 2001
From: Kerfuffle <44031344+KerfuffleV2@users.noreply.github.com>
Date: Sun, 3 Sep 2023 04:38:43 -0600
Subject: gguf(python): Fix special vocab handling when id < 0 (#2984)

---
 gguf-py/gguf/gguf.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'gguf-py/gguf/gguf.py')

diff --git a/gguf-py/gguf/gguf.py b/gguf-py/gguf/gguf.py
index b1bc4205..d377cd56 100644
--- a/gguf-py/gguf/gguf.py
+++ b/gguf-py/gguf/gguf.py
@@ -801,7 +801,7 @@ class SpecialVocab:
             else:
                 continue
             for maybe_token_id in (atok.get('id') for atok in added_tokens if atok.get('content') == tc_content):
-                if isinstance(maybe_token_id, int):
+                if isinstance(maybe_token_id, int) and maybe_token_id >= 0:
                     self.special_token_ids[typ] = maybe_token_id
                 break
         return True
@@ -814,7 +814,7 @@ class SpecialVocab:
             config = json.load(f)
         for typ in self.special_token_types:
             maybe_token_id = config.get(f'{typ}_token_id')
-            if isinstance(maybe_token_id, int):
+            if isinstance(maybe_token_id, int) and maybe_token_id >= 0:
                 self.special_token_ids[typ] = maybe_token_id
         return True
 
-- 
cgit v1.2.3