summaryrefslogtreecommitdiff
path: root/ggml.c
diff options
context:
space:
mode:
Diffstat (limited to 'ggml.c')
-rw-r--r--ggml.c150
1 files changed, 146 insertions, 4 deletions
diff --git a/ggml.c b/ggml.c
index 00bbee50..7f9bff99 100644
--- a/ggml.c
+++ b/ggml.c
@@ -2,6 +2,7 @@
#define _GNU_SOURCE
#include "ggml.h"
+#include "ggml-quants-k.h"
#if defined(_MSC_VER) || defined(__MINGW32__)
#include <malloc.h> // using malloc.h with MSC/MINGW
@@ -1565,6 +1566,46 @@ static const quantize_fns_t quantize_fns[GGML_TYPE_COUNT] = {
.vec_dot_q = NULL, // TODO
.vec_dot_type = GGML_TYPE_Q8_1,
},
+ [GGML_TYPE_Q2_K] = {
+ .dequantize_row_q = (dequantize_row_q_t) dequantize_row_q2_k,
+ .quantize_row_q = quantize_row_q2_k,
+ .quantize_row_q_reference = (quantize_row_q_t) quantize_row_q2_k_reference,
+ .quantize_row_q_dot = quantize_row_q8_k,
+ .vec_dot_q = ggml_vec_dot_q2_k_q8_k,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ },
+ [GGML_TYPE_Q3_K] = {
+ .dequantize_row_q = (dequantize_row_q_t) dequantize_row_q3_k,
+ .quantize_row_q = quantize_row_q3_k,
+ .quantize_row_q_reference = (quantize_row_q_t) quantize_row_q3_k_reference,
+ .quantize_row_q_dot = quantize_row_q8_k,
+ .vec_dot_q = ggml_vec_dot_q3_k_q8_k,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ },
+ [GGML_TYPE_Q4_K] = {
+ .dequantize_row_q = (dequantize_row_q_t) dequantize_row_q4_k,
+ .quantize_row_q = quantize_row_q4_k,
+ .quantize_row_q_reference = (quantize_row_q_t) quantize_row_q4_k_reference,
+ .quantize_row_q_dot = quantize_row_q8_k,
+ .vec_dot_q = ggml_vec_dot_q4_k_q8_k,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ },
+ [GGML_TYPE_Q5_K] = {
+ .dequantize_row_q = (dequantize_row_q_t) dequantize_row_q5_k,
+ .quantize_row_q = quantize_row_q5_k,
+ .quantize_row_q_reference = (quantize_row_q_t) quantize_row_q5_k_reference,
+ .quantize_row_q_dot = quantize_row_q8_k,
+ .vec_dot_q = ggml_vec_dot_q5_k_q8_k,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ },
+ [GGML_TYPE_Q6_K] = {
+ .dequantize_row_q = (dequantize_row_q_t) dequantize_row_q6_k,
+ .quantize_row_q = quantize_row_q6_k,
+ .quantize_row_q_reference = (quantize_row_q_t) quantize_row_q6_k_reference,
+ .quantize_row_q_dot = quantize_row_q8_k,
+ .vec_dot_q = ggml_vec_dot_q6_k_q8_k,
+ .vec_dot_type = GGML_TYPE_Q8_K,
+ },
};
// For internal test use
@@ -3444,11 +3485,17 @@ static const int GGML_BLCK_SIZE[GGML_TYPE_COUNT] = {
[GGML_TYPE_Q5_1] = QK5_1,
[GGML_TYPE_Q8_0] = QK8_0,
[GGML_TYPE_Q8_1] = QK8_1,
+ [GGML_TYPE_Q2_K] = QK_K,
+ [GGML_TYPE_Q3_K] = QK_K,
+ [GGML_TYPE_Q4_K] = QK_K,
+ [GGML_TYPE_Q5_K] = QK_K,
+ [GGML_TYPE_Q6_K] = QK_K,
+ [GGML_TYPE_Q8_K] = QK_K,
[GGML_TYPE_I8] = 1,
[GGML_TYPE_I16] = 1,
[GGML_TYPE_I32] = 1,
};
-static_assert(GGML_TYPE_COUNT == 13, "GGML_BLCK_SIZE is outdated");
+static_assert(GGML_TYPE_COUNT == 19, "GGML_BLCK_SIZE is outdated");
static const size_t GGML_TYPE_SIZE[GGML_TYPE_COUNT] = {
[GGML_TYPE_F32] = sizeof(float),
@@ -3459,11 +3506,17 @@ static const size_t GGML_TYPE_SIZE[GGML_TYPE_COUNT] = {
[GGML_TYPE_Q5_1] = sizeof(block_q5_1),
[GGML_TYPE_Q8_0] = sizeof(block_q8_0),
[GGML_TYPE_Q8_1] = sizeof(block_q8_1),
+ [GGML_TYPE_Q2_K] = sizeof(block_q2_k),
+ [GGML_TYPE_Q3_K] = sizeof(block_q3_k),
+ [GGML_TYPE_Q4_K] = sizeof(block_q4_k),
+ [GGML_TYPE_Q5_K] = sizeof(block_q5_k),
+ [GGML_TYPE_Q6_K] = sizeof(block_q6_k),
+ [GGML_TYPE_Q8_K] = sizeof(block_q8_k),
[GGML_TYPE_I8] = sizeof(int8_t),
[GGML_TYPE_I16] = sizeof(int16_t),
[GGML_TYPE_I32] = sizeof(int32_t),
};
-static_assert(GGML_TYPE_COUNT == 13, "GGML_TYPE_SIZE is outdated");
+static_assert(GGML_TYPE_COUNT == 19, "GGML_TYPE_SIZE is outdated");
static const char * GGML_TYPE_NAME[GGML_TYPE_COUNT] = {
@@ -3475,11 +3528,17 @@ static const char * GGML_TYPE_NAME[GGML_TYPE_COUNT] = {
[GGML_TYPE_Q5_1] = "q5_1",
[GGML_TYPE_Q8_0] = "q8_0",
[GGML_TYPE_Q8_1] = "q8_1",
+ [GGML_TYPE_Q2_K] = "q2_k",
+ [GGML_TYPE_Q3_K] = "q3_k",
+ [GGML_TYPE_Q4_K] = "q4_k",
+ [GGML_TYPE_Q5_K] = "q5_k",
+ [GGML_TYPE_Q6_K] = "q6_k",
+ [GGML_TYPE_Q8_K] = "q8_k",
[GGML_TYPE_I8] = "i8",
[GGML_TYPE_I16] = "i16",
[GGML_TYPE_I32] = "i32",
};
-static_assert(GGML_TYPE_COUNT == 13, "GGML_TYPE_NAME is outdated");
+static_assert(GGML_TYPE_COUNT == 19, "GGML_TYPE_NAME is outdated");
static bool GGML_IS_QUANTIZED[GGML_TYPE_COUNT] = {
[GGML_TYPE_F32] = false,
@@ -3490,11 +3549,17 @@ static bool GGML_IS_QUANTIZED[GGML_TYPE_COUNT] = {
[GGML_TYPE_Q5_1] = true,
[GGML_TYPE_Q8_0] = true,
[GGML_TYPE_Q8_1] = true,
+ [GGML_TYPE_Q2_K] = true,
+ [GGML_TYPE_Q3_K] = true,
+ [GGML_TYPE_Q4_K] = true,
+ [GGML_TYPE_Q5_K] = true,
+ [GGML_TYPE_Q6_K] = true,
+ [GGML_TYPE_Q8_K] = true,
[GGML_TYPE_I8] = false,
[GGML_TYPE_I16] = false,
[GGML_TYPE_I32] = false,
};
-static_assert(GGML_TYPE_COUNT == 13, "GGML_IS_QUANTIZED is outdated");
+static_assert(GGML_TYPE_COUNT == 19, "GGML_IS_QUANTIZED is outdated");
static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
"NONE",
@@ -3808,6 +3873,11 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {
case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break;
case GGML_FTYPE_MOSTLY_Q5_1: wtype = GGML_TYPE_Q5_1; break;
case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break;
+ case GGML_FTYPE_MOSTLY_Q2_K: wtype = GGML_TYPE_Q2_K; break;
+ case GGML_FTYPE_MOSTLY_Q3_K: wtype = GGML_TYPE_Q3_K; break;
+ case GGML_FTYPE_MOSTLY_Q4_K: wtype = GGML_TYPE_Q4_K; break;
+ case GGML_FTYPE_MOSTLY_Q5_K: wtype = GGML_TYPE_Q5_K; break;
+ case GGML_FTYPE_MOSTLY_Q6_K: wtype = GGML_TYPE_Q6_K; break;
case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break;
case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break;
}
@@ -7623,6 +7693,11 @@ static void ggml_compute_forward_add(
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
{
ggml_compute_forward_add_q_f32(params, src0, src1, dst);
} break;
@@ -7926,6 +8001,11 @@ static void ggml_compute_forward_add1(
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
{
ggml_compute_forward_add1_q_f32(params, src0, src1, dst);
} break;
@@ -8048,6 +8128,11 @@ static void ggml_compute_forward_acc(
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
default:
{
GGML_ASSERT(false);
@@ -10148,6 +10233,11 @@ static void ggml_compute_forward_mul_mat(
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
{
ggml_compute_forward_mul_mat_q_f32(params, src0, src1, dst);
} break;
@@ -10331,6 +10421,11 @@ static void ggml_compute_forward_set(
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
default:
{
GGML_ASSERT(false);
@@ -10496,6 +10591,11 @@ static void ggml_compute_forward_get_rows(
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
{
ggml_compute_forward_get_rows_q(params, src0, src1, dst);
} break;
@@ -11042,6 +11142,12 @@ static void ggml_compute_forward_alibi(
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_Q8_K:
case GGML_TYPE_I8:
case GGML_TYPE_I16:
case GGML_TYPE_I32:
@@ -11113,6 +11219,12 @@ static void ggml_compute_forward_clamp(
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_Q8_K:
case GGML_TYPE_I8:
case GGML_TYPE_I16:
case GGML_TYPE_I32:
@@ -16152,6 +16264,36 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i
block_q8_0 * block = (block_q8_0*)dst + start / QK8_0;
result = ggml_quantize_q8_0(src + start, block, n, n, hist);
} break;
+ case GGML_TYPE_Q2_K:
+ {
+ GGML_ASSERT(start % QK_K == 0);
+ block_q2_k * block = (block_q2_k*)dst + start / QK_K;
+ result = ggml_quantize_q2_k(src + start, block, n, n, hist);
+ } break;
+ case GGML_TYPE_Q3_K:
+ {
+ GGML_ASSERT(start % QK_K == 0);
+ block_q3_k * block = (block_q3_k*)dst + start / QK_K;
+ result = ggml_quantize_q3_k(src + start, block, n, n, hist);
+ } break;
+ case GGML_TYPE_Q4_K:
+ {
+ GGML_ASSERT(start % QK_K == 0);
+ block_q4_k * block = (block_q4_k*)dst + start / QK_K;
+ result = ggml_quantize_q4_k(src + start, block, n, n, hist);
+ } break;
+ case GGML_TYPE_Q5_K:
+ {
+ GGML_ASSERT(start % QK_K == 0);
+ block_q5_k * block = (block_q5_k*)dst + start / QK_K;
+ result = ggml_quantize_q5_k(src + start, block, n, n, hist);
+ } break;
+ case GGML_TYPE_Q6_K:
+ {
+ GGML_ASSERT(start % QK_K == 0);
+ block_q6_k * block = (block_q6_k*)dst + start / QK_K;
+ result = ggml_quantize_q6_k(src + start, block, n, n, hist);
+ } break;
default:
assert(false);
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-16 03:34:45 +0000