From b762db7c9264199c2d0f66e7d63e3b4884f3fc0c Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Thu, 27 Feb 2025 16:40:49 +0200 Subject: Option to use MLA without a transposed cache (#235) The `-mla` command line option turns into an int from a bool. mla = 0: use standard attention mla = 1: use MLA with transposed cache mla > 1: use MLA without transposed cache Co-authored-by: Iwan Kawrakow --- common/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'common/common.h') diff --git a/common/common.h b/common/common.h index 152fd1cf..ef5175f3 100644 --- a/common/common.h +++ b/common/common.h @@ -175,7 +175,7 @@ struct gpt_params { bool simple_io = false; // improves compatibility with subprocesses and limited consoles bool cont_batching = true; // insert new sequences for decoding on-the-fly bool flash_attn = false; // flash attention - bool mla_attn = false; // MLA + int mla_attn = false; // MLA 0: standard attention, 1: MLA with K and transposed V cache, 2: MLA with just K cache bool fused_moe_up_gate = false; // fused up*unary(gate) op for MoE models bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix -- cgit v1.2.3