summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/llama.h2
-rw-r--r--src/llama.cpp386
2 files changed, 348 insertions, 40 deletions
diff --git a/include/llama.h b/include/llama.h
index 3a81dce4..c21671c6 100644
--- a/include/llama.h
+++ b/include/llama.h
@@ -1046,6 +1046,8 @@ extern "C" {
bool add_ass,
char * buf,
int32_t length);
+ // Get list of built-in chat templates
+ LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len);
//
// Grammar
diff --git a/src/llama.cpp b/src/llama.cpp
index 29bd14af..c2bc5cc0 100644
--- a/src/llama.cpp
+++ b/src/llama.cpp
@@ -1364,6 +1364,76 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
},
};
+enum llm_chat_template {
+ LLM_CHAT_TEMPLATE_CHATML,
+ LLM_CHAT_TEMPLATE_LLAMA_2,
+ LLM_CHAT_TEMPLATE_LLAMA_2_SYS,
+ LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS,
+ LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP,
+ LLM_CHAT_TEMPLATE_MISTRAL_V1,
+ LLM_CHAT_TEMPLATE_MISTRAL_V3,
+ LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN,
+ LLM_CHAT_TEMPLATE_MISTRAL_V7,
+ LLM_CHAT_TEMPLATE_PHI_3,
+ LLM_CHAT_TEMPLATE_FALCON_3,
+ LLM_CHAT_TEMPLATE_ZEPHYR,
+ LLM_CHAT_TEMPLATE_MONARCH,
+ LLM_CHAT_TEMPLATE_GEMMA,
+ LLM_CHAT_TEMPLATE_ORION,
+ LLM_CHAT_TEMPLATE_OPENCHAT,
+ LLM_CHAT_TEMPLATE_VICUNA,
+ LLM_CHAT_TEMPLATE_VICUNA_ORCA,
+ LLM_CHAT_TEMPLATE_DEEPSEEK,
+ LLM_CHAT_TEMPLATE_DEEPSEEK_2,
+ LLM_CHAT_TEMPLATE_DEEPSEEK_3,
+ LLM_CHAT_TEMPLATE_COMMAND_R,
+ LLM_CHAT_TEMPLATE_LLAMA_3,
+ LLM_CHAT_TEMPLATE_CHATGML_3,
+ LLM_CHAT_TEMPLATE_CHATGML_4,
+ LLM_CHAT_TEMPLATE_MINICPM,
+ LLM_CHAT_TEMPLATE_EXAONE_3,
+ LLM_CHAT_TEMPLATE_RWKV_WORLD,
+ LLM_CHAT_TEMPLATE_GRANITE,
+ LLM_CHAT_TEMPLATE_GIGACHAT,
+ LLM_CHAT_TEMPLATE_MEGREZ,
+ LLM_CHAT_TEMPLATE_UNKNOWN,
+};
+
+static const std::map<std::string, llm_chat_template> LLM_CHAT_TEMPLATES = {
+ { "chatml", LLM_CHAT_TEMPLATE_CHATML },
+ { "llama2", LLM_CHAT_TEMPLATE_LLAMA_2 },
+ { "llama2-sys", LLM_CHAT_TEMPLATE_LLAMA_2_SYS },
+ { "llama2-sys-bos", LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS },
+ { "llama2-sys-strip", LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP },
+ { "mistral-v1", LLM_CHAT_TEMPLATE_MISTRAL_V1 },
+ { "mistral-v3", LLM_CHAT_TEMPLATE_MISTRAL_V3 },
+ { "mistral-v3-tekken", LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN },
+ { "mistral-v7", LLM_CHAT_TEMPLATE_MISTRAL_V7 },
+ { "phi3", LLM_CHAT_TEMPLATE_PHI_3 },
+ { "falcon3", LLM_CHAT_TEMPLATE_FALCON_3 },
+ { "zephyr", LLM_CHAT_TEMPLATE_ZEPHYR },
+ { "monarch", LLM_CHAT_TEMPLATE_MONARCH },
+ { "gemma", LLM_CHAT_TEMPLATE_GEMMA },
+ { "orion", LLM_CHAT_TEMPLATE_ORION },
+ { "openchat", LLM_CHAT_TEMPLATE_OPENCHAT },
+ { "vicuna", LLM_CHAT_TEMPLATE_VICUNA },
+ { "vicuna-orca", LLM_CHAT_TEMPLATE_VICUNA_ORCA },
+ { "deepseek", LLM_CHAT_TEMPLATE_DEEPSEEK },
+ { "deepseek2", LLM_CHAT_TEMPLATE_DEEPSEEK_2 },
+ { "deepseek3", LLM_CHAT_TEMPLATE_DEEPSEEK_3 },
+ { "command-r", LLM_CHAT_TEMPLATE_COMMAND_R },
+ { "llama3", LLM_CHAT_TEMPLATE_LLAMA_3 },
+ { "chatglm3", LLM_CHAT_TEMPLATE_CHATGML_3 },
+ { "chatglm4", LLM_CHAT_TEMPLATE_CHATGML_4 },
+ { "minicpm", LLM_CHAT_TEMPLATE_MINICPM },
+ { "exaone3", LLM_CHAT_TEMPLATE_EXAONE_3 },
+ { "rwkv-world", LLM_CHAT_TEMPLATE_RWKV_WORLD },
+ { "granite", LLM_CHAT_TEMPLATE_GRANITE },
+ { "gigachat", LLM_CHAT_TEMPLATE_GIGACHAT },
+ { "megrez", LLM_CHAT_TEMPLATE_MEGREZ },
+};
+
+
static llm_arch llm_arch_from_string(const std::string & name) {
for (const auto & kv : LLM_ARCH_NAMES) { // NOLINT
if (kv.second == name) {
@@ -19568,18 +19638,116 @@ int32_t llama_detokenize(
// chat templates
//
-// Simple version of "llama_apply_chat_template" that only works with strings
-// This function uses heuristic checks to determine commonly used template. It is not a jinja parser.
+static llm_chat_template llama_chat_detect_template(const std::string & tmpl) {
+ if (auto it = LLM_CHAT_TEMPLATES.find(tmpl); it != LLM_CHAT_TEMPLATES.end()) {
+ return it->second;
+ }
+ auto tmpl_contains = [&tmpl](const char * haystack) -> bool {
+ return tmpl.find(haystack) != std::string::npos;
+ };
+ if (tmpl_contains("<|im_start|>")) {
+ return LLM_CHAT_TEMPLATE_CHATML;
+ } else if (tmpl.find("mistral") == 0 || tmpl_contains("[INST]")) {
+ if (tmpl_contains("[SYSTEM_PROMPT]")) {
+ return LLM_CHAT_TEMPLATE_MISTRAL_V7;
+ } else if (
+ // catches official 'v1' template
+ tmpl_contains("' [INST] ' + system_message")
+ // catches official 'v3' and 'v3-tekken' templates
+ || tmpl_contains("[AVAILABLE_TOOLS]")
+ ) {
+ // Official mistral 'v1', 'v3' and 'v3-tekken' templates
+ // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/chat_templates.md
+ // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/templates.md
+ if (tmpl_contains(" [INST]")) {
+ return LLM_CHAT_TEMPLATE_MISTRAL_V1;
+ } else if (tmpl_contains("\"[INST]\"")) {
+ return LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN;
+ }
+ return LLM_CHAT_TEMPLATE_MISTRAL_V3;
+ } else {
+ // llama2 template and its variants
+ // [variant] support system message
+ // See: https://huggingface.co/blog/llama2#how-to-prompt-llama-2
+ bool support_system_message = tmpl_contains("<<SYS>>");
+ bool add_bos_inside_history = tmpl_contains("bos_token + '[INST]");
+ bool strip_message = tmpl_contains("content.strip()");
+ if (strip_message) {
+ return LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP;
+ } else if (add_bos_inside_history) {
+ return LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS;
+ } else if (support_system_message) {
+ return LLM_CHAT_TEMPLATE_LLAMA_2_SYS;
+ } else {
+ return LLM_CHAT_TEMPLATE_LLAMA_2;
+ }
+ }
+ } else if (tmpl_contains("<|assistant|>") && tmpl_contains("<|end|>")) {
+ return LLM_CHAT_TEMPLATE_PHI_3;
+ } else if (tmpl_contains("<|assistant|>") && tmpl_contains("<|user|>")) {
+ return LLM_CHAT_TEMPLATE_FALCON_3;
+ } else if (tmpl_contains("<|user|>") && tmpl_contains("<|endoftext|>")) {
+ return LLM_CHAT_TEMPLATE_ZEPHYR;
+ } else if (tmpl_contains("bos_token + message['role']")) {
+ return LLM_CHAT_TEMPLATE_MONARCH;
+ } else if (tmpl_contains("<start_of_turn>")) {
+ return LLM_CHAT_TEMPLATE_GEMMA;
+ } else if (tmpl_contains("'\\n\\nAssistant: ' + eos_token")) {
+ // OrionStarAI/Orion-14B-Chat
+ return LLM_CHAT_TEMPLATE_ORION;
+ } else if (tmpl_contains("GPT4 Correct ")) {
+ // openchat/openchat-3.5-0106
+ return LLM_CHAT_TEMPLATE_OPENCHAT;
+ } else if (tmpl_contains("USER: ") && tmpl_contains("ASSISTANT: ")) {
+ // eachadea/vicuna-13b-1.1 (and Orca variant)
+ if (tmpl_contains("SYSTEM: ")) {
+ return LLM_CHAT_TEMPLATE_VICUNA_ORCA;
+ }
+ return LLM_CHAT_TEMPLATE_VICUNA;
+ } else if (tmpl_contains("### Instruction:") && tmpl_contains("<|EOT|>")) {
+ // deepseek-ai/deepseek-coder-33b-instruct
+ return LLM_CHAT_TEMPLATE_DEEPSEEK;
+ } else if (tmpl_contains("<|START_OF_TURN_TOKEN|>") && tmpl_contains("<|USER_TOKEN|>")) {
+ // CohereForAI/c4ai-command-r-plus
+ return LLM_CHAT_TEMPLATE_COMMAND_R;
+ } else if (tmpl_contains("<|start_header_id|>") && tmpl_contains("<|end_header_id|>")) {
+ return LLM_CHAT_TEMPLATE_LLAMA_3;
+ } else if (tmpl_contains("[gMASK]sop")) {
+ // chatglm3-6b
+ return LLM_CHAT_TEMPLATE_CHATGML_3;
+ } else if (tmpl_contains("[gMASK]<sop>")) {
+ return LLM_CHAT_TEMPLATE_CHATGML_4;
+ } else if (tmpl_contains(LU8("<用户>"))) {
+ // MiniCPM-3B-OpenHermes-2.5-v2-GGUF
+ return LLM_CHAT_TEMPLATE_MINICPM;
+ } else if (tmpl_contains("'Assistant: ' + message['content'] + eos_token")) {
+ return LLM_CHAT_TEMPLATE_DEEPSEEK_2;
+ } else if (tmpl_contains(LU8("<｜Assistant｜>")) && tmpl_contains(LU8("<｜User｜>")) && tmpl_contains(LU8("<｜end▁of▁sentence｜>"))) {
+ // original: if (tmpl_contains(LU8("'<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>'"))) {
+ return LLM_CHAT_TEMPLATE_DEEPSEEK_3;
+ } else if (tmpl_contains("[|system|]") && tmpl_contains("[|assistant|]") && tmpl_contains("[|endofturn|]")) {
+ // ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb
+ // EXAONE-3.0-7.8B-Instruct
+ return LLM_CHAT_TEMPLATE_EXAONE_3;
+ } else if (tmpl_contains("rwkv-world")) {
+ return LLM_CHAT_TEMPLATE_RWKV_WORLD;
+ } else if (tmpl_contains("<|start_of_role|>")) {
+ return LLM_CHAT_TEMPLATE_GRANITE;
+ } else if (tmpl_contains("message['role'] + additional_special_tokens[0] + message['content'] + additional_special_tokens[1]")) {
+ return LLM_CHAT_TEMPLATE_GIGACHAT;
+ } else if (tmpl_contains("<|role_start|>")) {
+ return LLM_CHAT_TEMPLATE_MEGREZ;
+ }
+ return LLM_CHAT_TEMPLATE_UNKNOWN;
+}
+
static int32_t llama_chat_apply_template_internal(
- const std::string & tmpl,
+ const llm_chat_template tmpl,
const std::vector<const llama_chat_message *> & chat,
std::string & dest, bool add_ass) {
// Taken from the research: https://github.com/ggerganov/llama.cpp/issues/5527
std::stringstream ss;
- auto tmpl_contains = [&tmpl](std::string haystack) -> bool {
- return tmpl.find(haystack) != std::string::npos;
- };
- if (tmpl == "chatml" || tmpl_contains("<|im_start|>")) {
+ if (tmpl == LLM_CHAT_TEMPLATE_CHATML) {
// chatml template
for (auto message : chat) {
ss << "<|im_start|>" << message->role << "\n" << message->content << "<|im_end|>\n";
@@ -19587,16 +19755,59 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "<|im_start|>assistant\n";
}
- } else if (tmpl == "llama2" || tmpl == "mistral" || tmpl_contains("[INST]")) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V7) {
+ // Official mistral 'v7' template
+ // See: https://huggingface.co/mistralai/Mistral-Large-Instruct-2411#basic-instruct-template-v7
+ for (auto message : chat) {
+ std::string role(message->role);
+ std::string content(message->content);
+ if (role == "system") {
+ ss << "[SYSTEM_PROMPT] " << content << "[/SYSTEM_PROMPT]";
+ } else if (role == "user") {
+ ss << "[INST] " << content << "[/INST]";
+ }
+ else {
+ ss << " " << content << "</s>";
+ }
+ }
+ } else if (tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V1
+ || tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3
+ || tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN) {
+ // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/chat_templates.md
+ // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/templates.md
+ std::string leading_space = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V1 ? " " : "";
+ std::string trailing_space = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN ? "" : " ";
+ bool trim_assistant_message = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3;
+ bool is_inside_turn = false;
+ for (auto message : chat) {
+ if (!is_inside_turn) {
+ ss << leading_space << "[INST]" << trailing_space;
+ is_inside_turn = true;
+ }
+ std::string role(message->role);
+ std::string content(message->content);
+ if (role == "system") {
+ ss << content << "\n\n";
+ } else if (role == "user") {
+ ss << content << leading_space << "[/INST]";
+ } else {
+ ss << trailing_space << (trim_assistant_message ? trim(content) : content) << "</s>";
+ is_inside_turn = false;
+ }
+ }
+ } else if (
+ tmpl == LLM_CHAT_TEMPLATE_LLAMA_2
+ || tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS
+ || tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS
+ || tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP) {
// llama2 template and its variants
// [variant] support system message
- bool support_system_message = tmpl_contains("<<SYS>>") || tmpl == "mistral";
- // [variant] space before + after response
- bool space_around_response = tmpl_contains("' ' + eos_token");
+ // See: https://huggingface.co/blog/llama2#how-to-prompt-llama-2
+ bool support_system_message = tmpl != LLM_CHAT_TEMPLATE_LLAMA_2;
// [variant] add BOS inside history
- bool add_bos_inside_history = tmpl_contains("bos_token + '[INST]");
+ bool add_bos_inside_history = tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS;
// [variant] trim spaces from the input message
- bool strip_message = tmpl_contains("content.strip()");
+ bool strip_message = tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP;
// construct the prompt
bool is_inside_turn = true; // skip BOS at the beginning
ss << "[INST] ";
@@ -19617,12 +19828,11 @@ static int32_t llama_chat_apply_template_internal(
} else if (role == "user") {
ss << content << " [/INST]";
} else {
- ss << (space_around_response ? " " : "") << content << (space_around_response ? " " : "") << "</s>";
+ ss << content << "</s>";
is_inside_turn = false;
}
}
- // llama2 templates seem to not care about "add_generation_prompt"
- } else if (tmpl == "phi3" || (tmpl_contains("<|assistant|>") && tmpl_contains("<|end|>"))) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_PHI_3) {
// Phi 3
for (auto message : chat) {
std::string role(message->role);
@@ -19631,7 +19841,16 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "<|assistant|>\n";
}
- } else if (tmpl == "zephyr" || tmpl_contains("<|user|>")) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_FALCON_3) {
+ // Falcon 3
+ for (auto message : chat) {
+ std::string role(message->role);
+ ss << "<|" << role << "|>\n" << message->content << "\n";
+ }
+ if (add_ass) {
+ ss << "<|assistant|>\n";
+ }
+ } else if (tmpl == LLM_CHAT_TEMPLATE_ZEPHYR) {
// zephyr template
for (auto message : chat) {
ss << "<|" << message->role << "|>" << "\n" << message->content << "<|endoftext|>\n";
@@ -19639,7 +19858,7 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "<|assistant|>\n";
}
- } else if (tmpl == "monarch" || tmpl_contains("bos_token + message['role']")) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_MONARCH) {
// mlabonne/AlphaMonarch-7B template (the <s> is included inside history)
for (auto message : chat) {
std::string bos = (message == chat.front()) ? "" : "<s>"; // skip BOS for first message
@@ -19648,7 +19867,7 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "<s>assistant\n";
}
- } else if (tmpl == "gemma" || tmpl == "gemma2" || tmpl_contains("<start_of_turn>")) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_GEMMA) {
// google/gemma-7b-it
std::string system_prompt = "";
for (auto message : chat) {
@@ -19670,7 +19889,7 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "<start_of_turn>model\n";
}
- } else if (tmpl == "orion" || tmpl_contains("'\\n\\nAssistant: ' + eos_token")) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_ORION) {
// OrionStarAI/Orion-14B-Chat
std::string system_prompt = "";
for (auto message : chat) {
@@ -19690,7 +19909,7 @@ static int32_t llama_chat_apply_template_internal(
ss << message->content << "</s>";
}
}
- } else if (tmpl == "openchat" || tmpl_contains("GPT4 Correct ")) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_OPENCHAT) {
// openchat/openchat-3.5-0106,
for (auto message : chat) {
std::string role(message->role);
@@ -19704,13 +19923,13 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "GPT4 Correct Assistant:";
}
- } else if (tmpl == "vicuna" || tmpl == "vicuna-orca" || (tmpl_contains("USER: ") && tmpl_contains("ASSISTANT: "))) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_VICUNA || tmpl == LLM_CHAT_TEMPLATE_VICUNA_ORCA) {
// eachadea/vicuna-13b-1.1 (and Orca variant)
for (auto message : chat) {
std::string role(message->role);
if (role == "system") {
// Orca-Vicuna variant uses a system prefix
- if (tmpl == "vicuna-orca" || tmpl_contains("SYSTEM: ")) {
+ if (tmpl == LLM_CHAT_TEMPLATE_VICUNA_ORCA) {
ss << "SYSTEM: " << message->content << "\n";
} else {
ss << message->content << "\n\n";
@@ -19724,7 +19943,7 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "ASSISTANT:";
}
- } else if (tmpl == "deepseek" || (tmpl_contains("### Instruction:") && tmpl_contains("<|EOT|>"))) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK) {
// deepseek-ai/deepseek-coder-33b-instruct
for (auto message : chat) {
std::string role(message->role);
@@ -19739,7 +19958,7 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "### Response:\n";
}
- } else if (tmpl == "command-r" || (tmpl_contains("<|START_OF_TURN_TOKEN|>") && tmpl_contains("<|USER_TOKEN|>"))) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_COMMAND_R) {
// CohereForAI/c4ai-command-r-plus
for (auto message : chat) {
std::string role(message->role);
@@ -19754,7 +19973,7 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>";
}
- } else if (tmpl == "llama3" || (tmpl_contains("<|start_header_id|>") && tmpl_contains("<|end_header_id|>"))) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_LLAMA_3) {
// Llama 3
for (auto message : chat) {
std::string role(message->role);
@@ -19763,7 +19982,7 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "<|start_header_id|>assistant<|end_header_id|>\n\n";
}
- } else if (tmpl == "chatglm3" || tmpl_contains("[gMASK]sop")) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_CHATGML_3) {
// chatglm3-6b
ss << "[gMASK]" << "sop";
for (auto message : chat) {
@@ -19773,7 +19992,7 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "<|assistant|>";
}
- } else if (tmpl == "chatglm4" || tmpl_contains("[gMASK]<sop>")) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_CHATGML_4) {
ss << "[gMASK]" << "<sop>";
for (auto message : chat) {
std::string role(message->role);
@@ -19782,7 +20001,7 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "<|assistant|>";
}
- } else if (tmpl == "minicpm" || tmpl_contains(LU8("<用户>"))) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_MINICPM) {
// MiniCPM-3B-OpenHermes-2.5-v2-GGUF
for (auto message : chat) {
std::string role(message->role);
@@ -19794,7 +20013,7 @@ static int32_t llama_chat_apply_template_internal(
ss << trim(message->content);
}
}
- } else if (tmpl == "deepseek2" || tmpl_contains("'Assistant: ' + message['content'] + eos_token")) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK_2) {
// DeepSeek-V2
for (auto message : chat) {
std::string role(message->role);
@@ -19809,7 +20028,7 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "Assistant:";
}
- } else if (tmpl == "deepseek3" || tmpl_contains(LU8("'<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>'"))) {
+ } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK_3) {
// DeepSeek-V3
for (auto message : chat) {
std::string role(message->role);
@@ -19824,6 +20043,81 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << LU8("<｜Assistant｜>");
}
+ } else if (tmpl == LLM_CHAT_TEMPLATE_EXAONE_3) {
+ // ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb
+ // EXAONE-3.0-7.8B-Instruct
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "system") {
+ ss << "[|system|]" << trim(message->content) << "[|endofturn|]\n";
+ } else if (role == "user") {
+ ss << "[|user|]" << trim(message->content) << "\n";
+ } else if (role == "assistant") {
+ ss << "[|assistant|]" << trim(message->content) << "[|endofturn|]\n";
+ }
+ }
+ if (add_ass) {
+ ss << "[|assistant|]";
+ }
+ } else if (tmpl == LLM_CHAT_TEMPLATE_RWKV_WORLD) {
+ // this template requires the model to have "\n\n" as EOT token
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "user") {
+ ss << "User: " << message->content << "\n\nAssistant:";
+ } else {
+ ss << message->content << "\n\n";
+ }
+ }
+ } else if (tmpl == LLM_CHAT_TEMPLATE_GRANITE) {
+ // IBM Granite template
+ for (const auto & message : chat) {
+ std::string role(message->role);
+ ss << "<|start_of_role|>" << role << "<|end_of_role|>";
+ if (role == "assistant_tool_call") {
+ ss << "<|tool_call|>";
+ }
+ ss << message->content << "<|end_of_text|>\n";
+ }
+ if (add_ass) {
+ ss << "<|start_of_role|>assistant<|end_of_role|>\n";
+ }
+ } else if (tmpl == LLM_CHAT_TEMPLATE_GIGACHAT) {
+ // GigaChat template
+ bool has_system = !chat.empty() && std::string(chat[0]->role) == "system";
+
+ // Handle system message if present
+ if (has_system) {
+ ss << "<s>" << chat[0]->content << "<|message_sep|>";
+ } else {
+ ss << "<s>";
+ }
+
+ // Process remaining messages
+ for (size_t i = has_system ? 1 : 0; i < chat.size(); i++) {
+ std::string role(chat[i]->role);
+ if (role == "user") {
+ ss << "user<|role_sep|>" << chat[i]->content << "<|message_sep|>"
+ << "available functions<|role_sep|>[]<|message_sep|>";
+ } else if (role == "assistant") {
+ ss << "assistant<|role_sep|>" << chat[i]->content << "<|message_sep|>";
+ }
+ }
+
+ // Add generation prompt if needed
+ if (add_ass) {
+ ss << "assistant<|role_sep|>";
+ }
+ } else if (tmpl == LLM_CHAT_TEMPLATE_MEGREZ) {
+ // Megrez template
+ for (auto message : chat) {
+ std::string role(message->role);
+ ss << "<|role_start|>" << role << "<|role_end|>" << message->content << "<|turn_end|>";
+ }
+
+ if (add_ass) {
+ ss << "<|role_start|>assistant<|role_end|>";
+ }
} else {
// template not supported
return -1;
@@ -19843,15 +20137,15 @@ int32_t llama_chat_apply_template(
std::string curr_tmpl(tmpl == nullptr ? "" : tmpl);
if (tmpl == nullptr) {
GGML_ASSERT(model != nullptr);
- // load template from model
- std::vector<char> model_template(2048, 0); // longest known template is about 1200 bytes
- std::string template_key = "tokenizer.chat_template";
- int32_t res = llama_model_meta_val_str(model, template_key.c_str(), model_template.data(), model_template.size());
- if (res < 0) {
+
+ // load template from model, if available
+ const auto & it = model->gguf_kv.find("tokenizer.chat_template");
+ if (it != model->gguf_kv.end() && it->second.size() > 0) {
+ curr_tmpl = it->second;
+ }
+ else {
// worst case: there is no information about template, we will use chatml by default
- curr_tmpl = "chatml"; // see llama_chat_apply_template_internal
- } else {
- curr_tmpl = std::string(model_template.data(), model_template.size());
+ curr_tmpl = "chatml"; // see llama_chat_apply_template_internal
}
}
@@ -19863,7 +20157,11 @@ int32_t llama_chat_apply_template(
}
std::string formatted_chat;
- int32_t res = llama_chat_apply_template_internal(curr_tmpl, chat_vec, formatted_chat, add_ass);
+ llm_chat_template detected_tmpl = llama_chat_detect_template(curr_tmpl);
+ if (detected_tmpl == LLM_CHAT_TEMPLATE_UNKNOWN) {
+ return -1;
+ }
+ int32_t res = llama_chat_apply_template_internal(detected_tmpl, chat_vec, formatted_chat, add_ass);
if (res < 0) {
return res;
}
@@ -19873,6 +20171,14 @@ int32_t llama_chat_apply_template(
return res;
}
+int32_t llama_chat_builtin_templates(const char ** output, size_t len) {
+ auto it = LLM_CHAT_TEMPLATES.begin();
+ for (size_t i = 0; i < std::min(len, LLM_CHAT_TEMPLATES.size()); i++) {
+ output[i] = it->first.c_str();
+ std::advance(it, 1);
+ }
+ return (int32_t) LLM_CHAT_TEMPLATES.size();
+}
//
// grammar
//
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-17 20:08:24 +0000