From 910a13409463f7aedb0a92be013a1b9bb50f4859 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Sun, 13 Oct 2024 13:34:30 +0300 Subject: IQ2_KS: 2.1875 bpw non-linear quantization (#85) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Experimenting * iq2k: Try make_qx_quants for the scale Slightly better for LLaMA-3.1, Gemma-2, slightly worse for Qwen2.5 * iq2k with make_qx_quants: adjust scale * iq2ks: basics * iq2_ks: CUDA works * iq2_ks: WIP * iq2_ks: WIP * iq2_ks: Zen4 * iq2_ks: AVX2 * iq2_ks: scalar dot product * iq2_ks: ARM_NEON * iq2_ks: Metal * iq2_ks: faster Metal LLaMA-3.1-8B: PP-512 = 475.22 ± 0.37 t/s TG-128 = 45.32 ± 0.03 t/s --------- Co-authored-by: Iwan Kawrakow --- ggml/src/ggml-common.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'ggml/src/ggml-common.h') diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index 7eaf7437..3a7b8989 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -455,6 +455,13 @@ typedef struct { } block_iq2_k; static_assert(sizeof(block_iq2_k) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/32 + QK_K/4, "wrong iq2_k block size/padding"); +typedef struct { + uint16_t extra; + uint8_t scales[QK_K/64]; + uint8_t qs[QK_K/4]; +} block_iq2_ks; +static_assert(sizeof(block_iq2_ks) == sizeof(uint16_t) + QK_K/64 + QK_K/4, "wrong iq2_ks block size/padding"); + typedef struct { ggml_half d; uint16_t extra; -- cgit v1.2.3