Q3_K_R4 (#134)

* q3_k_r4: Zen4 works, but not as good as it should be

238 t/s, so sloghtly slower than q6_k_r4.

* q3_k_r4: NEON

We get PP-512(LLaMA-3.1-8B) = 106.9 t/s.
This is 1.93X faster than q3_K_S!

---------

Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>

1 files changed, 9 insertions, 0 deletions

diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h
index 2d73d3f8..bc34718e 100644
--- a/ggml/src/ggml-common.h
+++ b/ggml/src/ggml-common.h
@@ -288,6 +288,15 @@ typedef struct {
 } block_q3_K;
 static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding");
 
+typedef struct {
+    ggml_half d[4];             // super-block scales
+    uint8_t scales_h[QK_K/16];  // scales quantized with 6 bits (high 2 bits)
+    uint8_t scales_l[QK_K/8];   // scales quantized with 6 bits (low  4 bits)
+    uint8_t qh[QK_K/2];         // quants - high bit
+    uint8_t qs[QK_K];           // quants - low 2 bits
+} block_q3_k_r4;
+static_assert(sizeof(block_q3_k_r4) == 4*sizeof(ggml_half) + QK_K/16 + QK_K/8 + QK_K/2 + QK_K, "wrong q3_k_r4 block size/padding");
+
 // 4-bit quantization
 // 8 blocks of 32 elements each
 // weight is represented as x = a * q + b