From 905d87b70aa189623d500a28602d7a3a755a4769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sat, 13 May 2023 15:38:36 +0200 Subject: ggml : GPU-accelerated token generation (#1412) * CUDA kernel for q4_0 dequant. + mat. vec. mult. * Added q4_1 via template * Added missing __syncthreads(); * --gpu_layers -> --gpu-layers * Shorter dequantize_mul_mat_vec line * q5_0 dequantize_mul_mat kernel * More readable dequantize_mul_mat_vec logic * dequantize_mul_mat_vec kernels for q5_1, q8_0, f16 * llama : offload "output" tensor to GPU too + coding style fixes --------- Co-authored-by: Georgi Gerganov --- ggml-cuda.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'ggml-cuda.h') diff --git a/ggml-cuda.h b/ggml-cuda.h index f7d6a8bc..4e2c2428 100644 --- a/ggml-cuda.h +++ b/ggml-cuda.h @@ -14,6 +14,8 @@ void ggml_cuda_mul_mat(const struct ggml_tensor * src0, const struct ggml_tens void * ggml_cuda_host_malloc(size_t size); void ggml_cuda_host_free(void * ptr); +void ggml_cuda_transform_tensor(struct ggml_tensor * tensor); + #ifdef __cplusplus } #endif -- cgit v1.2.3