diff options
Diffstat (limited to 'convert-persimmon-to-gguf.py')
| -rwxr-xr-x | convert-persimmon-to-gguf.py | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/convert-persimmon-to-gguf.py b/convert-persimmon-to-gguf.py index 69be17f9..aba57542 100755 --- a/convert-persimmon-to-gguf.py +++ b/convert-persimmon-to-gguf.py @@ -99,6 +99,7 @@ def main(): tokens, scores, toktypes = _get_sentencepiece_tokenizer_info(args.model_dir) gguf_writer.add_tokenizer_model('llama') + gguf_writer.add_tokenizer_pre('default') gguf_writer.add_token_list(tokens) gguf_writer.add_token_scores(scores) gguf_writer.add_token_types(toktypes) |