From be55134a535f7218c53f39211755b1c7550851b2 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Thu, 28 Mar 2024 11:44:36 -0400 Subject: convert : refactor vocab selection logic (#6355) --- llama.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'llama.h') diff --git a/llama.h b/llama.h index 1fe4af49..f061d014 100644 --- a/llama.h +++ b/llama.h @@ -60,9 +60,9 @@ extern "C" { enum llama_vocab_type { LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab - LLAMA_VOCAB_TYPE_SPM = 1, // SentencePiece - LLAMA_VOCAB_TYPE_BPE = 2, // Byte Pair Encoding - LLAMA_VOCAB_TYPE_WPM = 3, // WordPiece + LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback + LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE + LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece }; // note: these values should be synchronized with ggml_rope -- cgit v1.2.3