diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/test-tokenizer-0-falcon.py | 2 | ||||
| -rw-r--r-- | tests/test-tokenizer-0-llama.py | 2 |
2 files changed, 0 insertions, 4 deletions
diff --git a/tests/test-tokenizer-0-falcon.py b/tests/test-tokenizer-0-falcon.py index cf65a3f6..65e1c0db 100644 --- a/tests/test-tokenizer-0-falcon.py +++ b/tests/test-tokenizer-0-falcon.py @@ -1,7 +1,5 @@ # tests with BPE tokenizer -import os -import sys import argparse from transformers import AutoTokenizer diff --git a/tests/test-tokenizer-0-llama.py b/tests/test-tokenizer-0-llama.py index 078f680b..21df8e6e 100644 --- a/tests/test-tokenizer-0-llama.py +++ b/tests/test-tokenizer-0-llama.py @@ -1,7 +1,5 @@ # tests with SPM tokenizer -import os -import sys import argparse from sentencepiece import SentencePieceProcessor |