summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGeorgi Gerganov <ggerganov@gmail.com>2024-01-16 20:59:31 +0200
committerGeorgi Gerganov <ggerganov@gmail.com>2024-01-16 20:59:31 +0200
commit5c999609013a30c06e6fd28be8db5c2074bcc196 (patch)
tree830685b7fa14f1c0f95357b89b7cb830ed51009f
parentbee938da74c33f42242c3a1058ac0a0a6eeef531 (diff)
py : remove unnecessary hasattr (#4903)
-rwxr-xr-xconvert-hf-to-gguf.py9
1 files changed, 4 insertions, 5 deletions
diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py
index b133f3b4..1178d63a 100755
--- a/convert-hf-to-gguf.py
+++ b/convert-hf-to-gguf.py
@@ -266,11 +266,10 @@ class Model:
toktypes.append(gguf.TokenType.USER_DEFINED)
elif reverse_vocab[i] in added_vocab:
tokens.append(reverse_vocab[i])
- if hasattr(tokenizer, "added_tokens_decoder"):
- if tokenizer.added_tokens_decoder[i].special:
- toktypes.append(gguf.TokenType.CONTROL)
- else:
- toktypes.append(gguf.TokenType.USER_DEFINED)
+ if tokenizer.added_tokens_decoder[i].special:
+ toktypes.append(gguf.TokenType.CONTROL)
+ else:
+ toktypes.append(gguf.TokenType.USER_DEFINED)
else:
tokens.append(reverse_vocab[i])
toktypes.append(gguf.TokenType.NORMAL)