From 7a32fcb3b29f4db8aed8a85dc58eb958fb118153 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 25 Apr 2023 23:40:51 +0300 Subject: ggml : add Q8_0 quantization format (rename the old one to Q8_1) (ARM NEON) (#1179) * ggml : add Q8_0 quantization format (rename the old one to Q8_1) * tests : fix test-quantize-fns * ggml : finalize Q8_0 implementation * ggml : use q4_0_q8_0 and q4_2_q8_0 * ggml : fix Q8_0 dot product bug (ARM) * ggml : Q8_0 unroll x2 * ggml : fix bug - using wrong block type * ggml : extend quantize_fns_t with "vec_dot_type" * ggml : fix Q8_0 to use 255 values out of 256 * ggml : fix assert using wrong QK4_2 instead of QK4_3 --- ggml.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'ggml.h') diff --git a/ggml.h b/ggml.h index 27589078..8300a0c6 100644 --- a/ggml.h +++ b/ggml.h @@ -223,6 +223,7 @@ extern "C" { GGML_TYPE_Q4_2 = 4, GGML_TYPE_Q4_3 = 5, GGML_TYPE_Q8_0 = 6, + GGML_TYPE_Q8_1 = 7, GGML_TYPE_I8, GGML_TYPE_I16, GGML_TYPE_I32, @@ -832,6 +833,7 @@ extern "C" { GGML_API size_t ggml_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist); GGML_API size_t ggml_quantize_q4_2(const float * src, void * dst, int n, int k, int64_t * hist); GGML_API size_t ggml_quantize_q4_3(const float * src, void * dst, int n, int k, int64_t * hist); + GGML_API size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t * hist); GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist); @@ -876,6 +878,7 @@ extern "C" { quantize_row_q_t quantize_row_q_reference; quantize_row_q_t quantize_row_q_dot; vec_dot_q_t vec_dot_q; + enum ggml_type vec_dot_type; } quantize_fns_t; quantize_fns_t ggml_internal_get_quantize_fn(size_t i); -- cgit v1.2.3