From 0999f77e5b1a97164ee0218f5fc118fe1649b0a3 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Tue, 1 Oct 2024 15:52:35 +0300 Subject: Fix Q5_0 flash attention (#75) When I changed iqk_mul_mat to use type-1 dot products for type-0 legacy quants, I forgot to also change the vec_dot_type when the dot product is done via ggml as in flash attention. This commit fixes it. Co-authored-by: Iwan Kawrakow --- ggml/src/ggml-quants.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'ggml/src/ggml-quants.c') diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index b0e70bcc..d6b1dc0a 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -4641,7 +4641,12 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { #if GGML_USE_IQK_MULMAT - if (iqk_mul_mat(nrc, nrc, n, GGML_TYPE_Q5_0, vx, bx, GGML_TYPE_Q8_0, vy, by, s, bs, 0, 1)) { +#ifdef __AVX2__ + const enum ggml_type vec_dot_type = GGML_TYPE_Q8_1; +#else + const enum ggml_type vec_dot_type = GGML_TYPE_Q8_0; +#endif + if (iqk_mul_mat(nrc, nrc, n, GGML_TYPE_Q5_0, vx, bx, vec_dot_type, vy, by, s, bs, 0, 1)) { return; } #endif -- cgit v1.2.3