From 7553989dd88749de028853f9c0ea39651aad92a3 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Fri, 10 Jan 2025 18:18:04 +0200 Subject: Be able to re-quantize MS BitNet I2_S models (#169) Co-authored-by: Iwan Kawrakow --- src/llama.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/llama.cpp') diff --git a/src/llama.cpp b/src/llama.cpp index 54b9b118..d330904f 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -15698,6 +15698,12 @@ static void llama_tensor_dequantize_internal( throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor->type))); } + if (tensor->type == GGML_TYPE_I2_S) { + // we need to dequantize the entire tensor for I2_S + qtype.to_float(tensor->data, f32_output, nelements); + return; + } + if (nthread < 2) { if (tensor->type == GGML_TYPE_F16) { ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor->data, f32_output, nelements); -- cgit v1.2.3