diff options
author | Sang-Kil Park <sang.park@42dot.ai> | 2024-02-07 13:28:00 +0900 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-02-06 23:28:00 -0500 |
commit | f68664ac241a6b5c233d8f1051eef20929b06008 (patch) | |
tree | b321257aae6e3706590791f69cc931a01350f8c7 | |
parent | 213d1439fadefe182f69c5f7e8dd3b4b6572ebcb (diff) |
convert : fix TypeError on GPT-2 vocab.json (#5288)
-rwxr-xr-x | convert.py | 4 |
1 files changed, 2 insertions, 2 deletions
@@ -334,9 +334,9 @@ class Params: class BpeVocab: def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None: self.bpe_tokenizer = json.loads(open(str(fname_tokenizer), encoding="utf-8").read()) - try: + if isinstance(self.bpe_tokenizer.get('model'), dict): self.vocab = self.bpe_tokenizer["model"]["vocab"] - except KeyError: + else: self.vocab = self.bpe_tokenizer added_tokens: dict[str, int] if fname_added_tokens is not None: |