diff options
| -rwxr-xr-x | convert-hf-to-gguf.py | 9 |
1 files changed, 4 insertions, 5 deletions
diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index b133f3b4..1178d63a 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -266,11 +266,10 @@ class Model: toktypes.append(gguf.TokenType.USER_DEFINED) elif reverse_vocab[i] in added_vocab: tokens.append(reverse_vocab[i]) - if hasattr(tokenizer, "added_tokens_decoder"): - if tokenizer.added_tokens_decoder[i].special: - toktypes.append(gguf.TokenType.CONTROL) - else: - toktypes.append(gguf.TokenType.USER_DEFINED) + if tokenizer.added_tokens_decoder[i].special: + toktypes.append(gguf.TokenType.CONTROL) + else: + toktypes.append(gguf.TokenType.USER_DEFINED) else: tokens.append(reverse_vocab[i]) toktypes.append(gguf.TokenType.NORMAL) |