From 33390c4b74fa52875d6028c5c9aaf84f17288c25 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Sun, 9 Feb 2025 09:14:52 +0200 Subject: Use Q8_K_128 for IQ1_S_R4 and IQ1_M_R4 matrix multiplications (#194) * iq1_s_r4: Use Q8_K_128 instead of Q8_1_X4 for gemm (AVX2/Zen4) * iq1_m_r4: Use Q8_K_128 instead of Q8_1_X4 for gemm (AVX2/Zen4) * iq1_s_r4: Use Q8_K_128 instead of Q8_1_X4 for gemm (Neon) * iq1_m_r4: Use Q8_K_128 instead of Q8_0_X4 for gemm (Neon) * Simdify q8_K128 quantization also on Neon * Cleanup --------- Co-authored-by: Iwan Kawrakow --- ggml/src/ggml-common.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'ggml/src/ggml-common.h') diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index 0d014c23..4308f0b9 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -377,15 +377,16 @@ typedef struct { } block_q8_K; static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding"); typedef struct { - float d; // delta + float d; // delta int8_t qs[64]; // quants } block_q8_K64; static_assert(sizeof(block_q8_K64) == sizeof(float) + 64, "wrong q8_K64 block size/padding"); typedef struct { float d; // delta + int16_t bsums[4]; // quant sums for blocks of 32 int8_t qs[128]; // quants } block_q8_K128; -static_assert(sizeof(block_q8_K128) == sizeof(float) + 128, "wrong q8_K128 block size/padding"); +static_assert(sizeof(block_q8_K128) == sizeof(float) + 4*sizeof(int16_t) + 128, "wrong q8_K128 block size/padding"); typedef struct { ggml_half d[8]; // delta -- cgit v1.2.3