IQ4_K_R4 (#138)

* iq4_k_r4: WIP

* iq4_k_r4: Zen4 and hopefully AVX2

On Zen4 we get PP-512(LLaMA-3.1-8B) = 232.6 t/s, up from 182.2 t/s
for iq4_k. Applying the extra shift costs a ~6 performance penalty.

* iq4_k_r4: AVX2

PP-512 = 227.60 t/s. The shifts are really costly.

* iq4_k_r4: NEON

We get PP-512(LLaMA-3.1-8B) = 108 t/s, up from 58.2 t/s for iq4_k.

---------

Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>

1 files changed, 9 insertions, 0 deletions

diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h
index 61e8dfd3..2cacc711 100644
--- a/ggml/src/ggml-common.h
+++ b/ggml/src/ggml-common.h
@@ -542,6 +542,15 @@ typedef struct {
 static_assert(sizeof(block_iq4_k) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/2 + 3*QK_K/64, "wrong iq4_k block size/padding");
 
 typedef struct {
+    ggml_half d[4];
+    uint8_t  extra[8];
+    uint8_t  scales_h[QK_K/16];
+    uint8_t  scales_l[QK_K/8];
+    uint8_t  qs[QK_K*2];
+} block_iq4_k_r4;
+static_assert(sizeof(block_iq4_k_r4) == 4*sizeof(block_iq4_k), "wrong iq4_k_r4 block size/padding");
+
+typedef struct {
     ggml_half d;
     uint16_t extra;
     uint8_t  scales_h[QK_K/64];