From 4d2fbde0cbbfc98200b59ed6fe5b32628a70c055 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Thu, 21 Nov 2024 07:12:11 +0100 Subject: MMQ for Q6_0 (#115) * MMQ for Q6_0 * Add Q6_0 MMQ to template generator --------- Co-authored-by: Iwan Kawrakow --- ggml/src/ggml-cuda/template-instances/generate_cu_files.py | 2 +- ggml/src/ggml-cuda/template-instances/mmq-instance-q6_0.cu | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q6_0.cu (limited to 'ggml/src/ggml-cuda/template-instances') diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py index 4f7489d5..3037aa96 100755 --- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py @@ -24,7 +24,7 @@ TYPES_MMQ = [ "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_Q2_K", "GGML_TYPE_Q3_K", "GGML_TYPE_Q4_K", "GGML_TYPE_Q5_K", "GGML_TYPE_Q6_K", "GGML_TYPE_IQ2_XXS", "GGML_TYPE_IQ2_XS", "GGML_TYPE_IQ2_S", "GGML_TYPE_IQ3_XXS", "GGML_TYPE_IQ3_S", - "GGML_TYPE_IQ1_S", "GGML_TYPE_IQ4_NL", "GGML_TYPE_IQ4_XS" + "GGML_TYPE_IQ1_S", "GGML_TYPE_IQ4_NL", "GGML_TYPE_IQ4_XS", "GGML_TYPE_Q6_0" ] SOURCE_MMQ = """// This file has been autogenerated by generate_cu_files.py, do not edit manually. diff --git a/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_0.cu b/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_0.cu new file mode 100644 index 00000000..8a728e6c --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_0.cu @@ -0,0 +1,5 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../mmq.cuh" + +DECL_MMQ_CASE(GGML_TYPE_Q6_0); -- cgit v1.2.3