diff options
| author | Johannes Gäßler <johannesg@5d6.de> | 2024-04-14 00:21:55 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-04-14 00:21:55 +0200 |
| commit | b5e7285baffb0da8a6619567b52d8e67de41291d (patch) | |
| tree | c2183198e28969e1b899a55fe0b633770cdafb5a | |
| parent | 4bd0f93e4ab4fe6682e7d0241c1bdec1397e954a (diff) | |
CUDA: fix matrix multiplication logic for tests (#6667)
| -rw-r--r-- | ggml-cuda.cu | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/ggml-cuda.cu b/ggml-cuda.cu index bff8ad9d..e62f45b6 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -1946,7 +1946,7 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor } else if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) { // KQV single-batch ggml_cuda_mul_mat_vec_nc(ctx, src0, src1, dst); - } else if (!split && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) { + } else if (!split && src0->type == GGML_TYPE_F16 && (src1->type == GGML_TYPE_F16 || fp16_performance_good) && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) { // KQ + KQV multi-batch ggml_cuda_mul_mat_batched_cublas(ctx, src0, src1, dst); } else if (use_dequantize_mul_mat_vec) { |