From a48e16324770bb829406d06e11be1df0c8a3b517 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Mon, 10 Mar 2025 16:19:09 +0200 Subject: DeepSeek imatrix stuff (#250) * This gives us ~20% TG speedup for DeepSeek on CUDA * Slightly better * Also do it for plain (not fused) mul_mat_id * Guard against numerical precision issues for MLA on CUDA * imatrix: wv_b <-> wkv_b --------- Co-authored-by: Iwan Kawrakow --- examples/imatrix/imatrix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'examples/imatrix/imatrix.cpp') diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp index 8006988c..d8a43049 100644 --- a/examples/imatrix/imatrix.cpp +++ b/examples/imatrix/imatrix.cpp @@ -195,7 +195,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * if (m_params.verbosity > 1) { printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, wname.c_str(), ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type); } - for (int row = 0; row < (int)src1->ne[1]; ++row) { + for (int row = 0; row < (int)(src1->ne[1]*src1->ne[2]); ++row) { const float * x = data + row * src1->ne[0]; for (int j = 0; j < (int)src1->ne[0]; ++j) { e.values[j] += x[j]*x[j]; -- cgit v1.2.3