summaryrefslogtreecommitdiff
path: root/ggml/src/ggml.c
diff options
context:
space:
mode:
authorKawrakow <iwankawrakow@gmail.com>2024-12-10 18:13:47 +0100
committerGitHub <noreply@github.com>2024-12-10 18:13:47 +0100
commita63a96b5aea031f9dabf1e7c8d5ee28af170e9e7 (patch)
tree01e5c72689e8daa72a53bbe069c8669655187a34 /ggml/src/ggml.c
parentc819fa651beb20c559f692801819e0447d0ead85 (diff)
Q5_K_R4 (#132)
* q5_k_r4: WIP * q5_k_r4: Zen4 and AVX2 We get PP-512(LLaMA-3.1-8B) = 248.3 t/s on Zen4. Q5_K_S has PP-512 = 190 t/s. * q5_k_r4: NEON We get PP-512(LLaMA-3.1-8B) = 96.1 t/s. --------- Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
Diffstat (limited to 'ggml/src/ggml.c')
-rw-r--r--ggml/src/ggml.c22
1 files changed, 22 insertions, 0 deletions
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
index b92c2352..53c51ba6 100644
--- a/ggml/src/ggml.c
+++ b/ggml/src/ggml.c
@@ -914,6 +914,19 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {
.nrows = 1,
.row_meta_size = 0,
},
+ [GGML_TYPE_Q5_K_R4] = {
+ .type_name = "q5_k_r4",
+ .blck_size = QK_K,
+ .type_size = sizeof(block_q5_K),
+ .is_quantized = true,
+ .to_float = (ggml_to_float_t) dequantize_row_q5_k_r4,
+ .from_float = quantize_row_q5_k_r4,
+ .from_float_ref = (ggml_from_float_t) quantize_row_q5_k_r4_ref,
+ .vec_dot = vec_dot_q5_k_r4_q8_k,
+ .vec_dot_type = GGML_TYPE_Q8_K32,
+ .nrows = 1,
+ .row_meta_size = 0,
+ },
[GGML_TYPE_Q6_K] = {
.type_name = "q6_K",
.blck_size = QK_K,
@@ -4048,6 +4061,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {
case GGML_FTYPE_MOSTLY_Q4_K: wtype = GGML_TYPE_Q4_K; break;
case GGML_FTYPE_MOSTLY_Q4_K_R4: wtype = GGML_TYPE_Q4_K_R4; break;
case GGML_FTYPE_MOSTLY_Q5_K: wtype = GGML_TYPE_Q5_K; break;
+ case GGML_FTYPE_MOSTLY_Q5_K_R4: wtype = GGML_TYPE_Q5_K_R4; break;
case GGML_FTYPE_MOSTLY_Q6_K: wtype = GGML_TYPE_Q6_K; break;
case GGML_FTYPE_MOSTLY_Q6_K_R4: wtype = GGML_TYPE_Q6_K_R4; break;
case GGML_FTYPE_MOSTLY_IQ2_XXS: wtype = GGML_TYPE_IQ2_XXS; break;
@@ -10580,6 +10594,7 @@ static void ggml_compute_forward_add(
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q4_K_R4:
case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q5_K_R4:
case GGML_TYPE_Q6_K:
case GGML_TYPE_Q6_K_R4:
case GGML_TYPE_IQ2_XXS:
@@ -11031,6 +11046,7 @@ static void ggml_compute_forward_add1(
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q4_K_R4:
case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q5_K_R4:
case GGML_TYPE_Q6_K:
case GGML_TYPE_Q6_K_R4:
case GGML_TYPE_IQ2_XXS:
@@ -11179,6 +11195,7 @@ static void ggml_compute_forward_acc(
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q4_K_R4:
case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q5_K_R4:
case GGML_TYPE_Q6_K:
case GGML_TYPE_Q6_K_R4:
case GGML_TYPE_IQ2_XXS:
@@ -14373,6 +14390,7 @@ static void ggml_compute_forward_out_prod(
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q4_K_R4:
case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q5_K_R4:
case GGML_TYPE_Q6_K:
case GGML_TYPE_Q6_K_R4:
case GGML_TYPE_IQ2_XXS:
@@ -14761,6 +14779,7 @@ static void ggml_compute_forward_set(
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q4_K_R4:
case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q5_K_R4:
case GGML_TYPE_Q6_K:
case GGML_TYPE_Q6_K_R4:
case GGML_TYPE_IQ2_XXS:
@@ -15043,6 +15062,7 @@ static void ggml_compute_forward_get_rows(
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q4_K_R4:
case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q5_K_R4:
case GGML_TYPE_Q6_K:
case GGML_TYPE_Q6_K_R4:
case GGML_TYPE_IQ2_XXS:
@@ -15652,6 +15672,7 @@ static void ggml_compute_forward_clamp(
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q4_K_R4:
case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q5_K_R4:
case GGML_TYPE_Q6_K:
case GGML_TYPE_Q6_K_R4:
case GGML_TYPE_IQ2_XXS:
@@ -22489,6 +22510,7 @@ size_t ggml_quantize_chunk(
case GGML_TYPE_Q4_K: result = quantize_q4_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_Q4_K_R4: result = quantize_q4_k_r4(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_Q5_K: result = quantize_q5_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
+ case GGML_TYPE_Q5_K_R4: result = quantize_q5_k_r4(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_Q6_K: result = quantize_q6_K(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_Q6_K_R4: result = quantize_q6_k_r4(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
case GGML_TYPE_IQ2_XXS: result = quantize_iq2_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;