diff options
| author | Kawrakow <iwankawrakow@gmail.com> | 2025-06-11 15:01:34 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2025-06-11 15:01:34 +0300 |
| commit | 3f54b4978672cc236bf38e707b24f036cb37f3f7 (patch) | |
| tree | 41221f8bfc21f1fa4689df962d98aff95fc2df68 /ggml/src/iqk/iqk_mul_mat.cpp | |
| parent | 69af3f5990327df4f8dcb05817825802b7e6bed8 (diff) | |
Faster iq1_s GEMM via repacking to Q8_0_R8 (#517)
TG is slightly faster too - 24.4 vs 23.1 t/s on the
Ryzen-5975WX
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
Diffstat (limited to 'ggml/src/iqk/iqk_mul_mat.cpp')
| -rw-r--r-- | ggml/src/iqk/iqk_mul_mat.cpp | 11 |
1 files changed, 6 insertions, 5 deletions
diff --git a/ggml/src/iqk/iqk_mul_mat.cpp b/ggml/src/iqk/iqk_mul_mat.cpp index 16896909..734631f0 100644 --- a/ggml/src/iqk/iqk_mul_mat.cpp +++ b/ggml/src/iqk/iqk_mul_mat.cpp @@ -236,11 +236,12 @@ struct MulMat { static inline ggml_type is_dequant_better(ggml_type type, int nrc_y) { #ifdef __AVX2__ switch (type) { - case GGML_TYPE_IQ2_KT: return nrc_y >= 32 ? GGML_TYPE_F32 : type; - case GGML_TYPE_IQ3_KT: return nrc_y >= 32 ? GGML_TYPE_F32 : type; - case GGML_TYPE_IQ4_KT: return nrc_y >= 32 ? GGML_TYPE_F32 : type; + case GGML_TYPE_IQ2_KT : return nrc_y >= 32 ? GGML_TYPE_F32 : type; + case GGML_TYPE_IQ3_KT : return nrc_y >= 32 ? GGML_TYPE_F32 : type; + case GGML_TYPE_IQ4_KT : return nrc_y >= 32 ? GGML_TYPE_F32 : type; case GGML_TYPE_IQ2_XXS: return nrc_y >= 32 ? GGML_TYPE_Q8_0_R8 : type; case GGML_TYPE_IQ3_XXS: return nrc_y >= 32 ? GGML_TYPE_Q8_0_R8 : type; + case GGML_TYPE_IQ1_S : return nrc_y >= 32 ? GGML_TYPE_Q8_0_R8 : type; default: break; } #else @@ -397,13 +398,13 @@ bool iqk_convert_repack(int typeA, int n, const void * vx, size_t bx, void * vy, //case GGML_TYPE_Q8_0_R8: //case GGML_TYPE_IQ4_NL_R4: // return iqk_set_kernels_legacy_quants(ne00, typeA, typeB, mm.funcs, mm.func16); - //case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_S: //case GGML_TYPE_IQ1_S_R4: //case GGML_TYPE_IQ1_M_R4: //case GGML_TYPE_IQ1_BN: //case GGML_TYPE_IQ2_BN: //case GGML_TYPE_IQ2_BN_R4: - // return iqk_set_kernels_1bit(ne00, typeA, typeB, mm.funcs, mm.func16); + return iqk_convert_1bit_q80_r8(typeA, n, vx, bx, vy, nrc_x); default: return false; |