From b762db7c9264199c2d0f66e7d63e3b4884f3fc0c Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Thu, 27 Feb 2025 16:40:49 +0200 Subject: Option to use MLA without a transposed cache (#235) The `-mla` command line option turns into an int from a bool. mla = 0: use standard attention mla = 1: use MLA with transposed cache mla > 1: use MLA without transposed cache Co-authored-by: Iwan Kawrakow --- include/llama.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/llama.h b/include/llama.h index beb6ecba..2b33701c 100644 --- a/include/llama.h +++ b/include/llama.h @@ -383,7 +383,7 @@ extern "C" { bool embeddings; // if true, extract embeddings (together with logits) bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU bool flash_attn; // whether to use flash attention [EXPERIMENTAL] - bool mla_attn; // whether to use MLA attention [EXPERIMENTAL] + int mla_attn; // whether to use MLA attention [EXPERIMENTAL] bool fused_moe_up_gate; // whether to use fused MoE up/down op [EXPERIMENTAL] // Abort callback -- cgit v1.2.3