summaryrefslogtreecommitdiff
path: root/ggml/src/iqk/iqk_quantize.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'ggml/src/iqk/iqk_quantize.cpp')
-rw-r--r--ggml/src/iqk/iqk_quantize.cpp604
1 files changed, 467 insertions, 137 deletions
diff --git a/ggml/src/iqk/iqk_quantize.cpp b/ggml/src/iqk/iqk_quantize.cpp
index 7722d630..9c502f07 100644
--- a/ggml/src/iqk/iqk_quantize.cpp
+++ b/ggml/src/iqk/iqk_quantize.cpp
@@ -414,6 +414,221 @@ void quantize_row_q8_K64(const float * x, void * y, int64_t k) {
}
//
+// ============================================== iq2_K
+//
+
+namespace {
+
+inline int best_index_iq2nl(const int8_t * values, float x) {
+ int idx = x < values[1] ? 0 : x > values[2] ? 2 : 1;
+ return x - values[idx] < values[idx+1] - x ? idx : idx + 1;
+}
+
+void quantize_row_iq2_k_impl(const float * x, void * vy, int n_per_row, const float * quant_weights) {
+
+ constexpr int kBlockSize = 16;
+
+ block_iq2_k * y = (block_iq2_k *)vy;
+
+ float scales[QK_K/kBlockSize];
+ float weight[kBlockSize];
+ float sumx[kBlockSize+1], sumw[kBlockSize+1];
+
+ std::array<std::pair<float,int>, kBlockSize> pairs;
+
+ const int8_t * shifted_values = iq2nl_values + 4;
+
+ for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) {
+
+ memset(&y[ibl], 0, sizeof(block_iq2_k));
+ y[ibl].d = GGML_FP32_TO_FP16(0.f);
+
+ const float * xbl = x + ibl*QK_K;
+ float sumx2 = 0;
+ for (int j = 0; j < QK_K; ++j) sumx2 += xbl[j]*xbl[j];
+ const float sigma2 = 1.5f*sumx2/QK_K;
+
+ uint16_t extra = 0;
+
+ float max_abs_scale = 0;
+
+ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) {
+ const float * xb = xbl + kBlockSize*ib;
+ if (quant_weights) {
+ const float * qw = quant_weights + ibl*QK_K + ib*kBlockSize;
+ for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
+ } else {
+ for (int j = 0; j < kBlockSize; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j];
+ }
+ for (int j = 0; j < kBlockSize; ++j) pairs[j] = {xb[j], j};
+ std::sort(pairs.begin(), pairs.end());
+ sumx[0] = sumw[0] = 0;
+ for (int j = 0; j < kBlockSize; ++j) {
+ int jj = pairs[j].second;
+ sumw[j+1] = sumw[j] + weight[jj];
+ sumx[j+1] = sumx[j] + weight[jj]*xb[jj];
+ }
+ float best = 0, d = 0;
+ bool is_shifted = false;
+ float sumqx, sumq2;
+ for (int i1 = 0; i1 < kBlockSize; ++i1) {
+ for (int i2 = i1; i2 < kBlockSize; ++i2) {
+ for (int i3 = i2; i3 < kBlockSize; ++i3) {
+ sumqx = (sumx[i1] - sumx[ 0])*iq2nl_values[0] + (sumx[i2] - sumx[i1])*iq2nl_values[1]
+ + (sumx[i3] - sumx[i2])*iq2nl_values[2] + (sumx[kBlockSize] - sumx[i3])*iq2nl_values[3];
+ sumq2 = (sumw[i1] - sumw[ 0])*iq2nl_values[0]*iq2nl_values[0] + (sumw[i2] - sumw[i1])*iq2nl_values[1]*iq2nl_values[1]
+ + (sumw[i3] - sumw[i2])*iq2nl_values[2]*iq2nl_values[2] + (sumw[kBlockSize] - sumw[i3])*iq2nl_values[3]*iq2nl_values[3];
+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
+ d = sumqx/sumq2; best = d*sumqx; is_shifted = false;
+ }
+ sumqx = (sumx[i1] - sumx[ 0])*shifted_values[0] + (sumx[i2] - sumx[i1])*shifted_values[1]
+ + (sumx[i3] - sumx[i2])*shifted_values[2] + (sumx[kBlockSize] - sumx[i3])*shifted_values[3];
+ sumq2 = (sumw[i1] - sumw[ 0])*shifted_values[0]*shifted_values[0] + (sumw[i2] - sumw[i1])*shifted_values[1]*shifted_values[1]
+ + (sumw[i3] - sumw[i2])*shifted_values[2]*shifted_values[2] + (sumw[kBlockSize] - sumw[i3])*shifted_values[3]*shifted_values[3];
+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
+ d = sumqx/sumq2; best = d*sumqx; is_shifted = true;
+ }
+ sumqx = (sumx[i1] - sumx[ 0])*iq2nl_values[3] + (sumx[i2] - sumx[i1])*iq2nl_values[2]
+ + (sumx[i3] - sumx[i2])*iq2nl_values[1] + (sumx[kBlockSize] - sumx[i3])*iq2nl_values[0];
+ sumq2 = (sumw[i1] - sumw[ 0])*iq2nl_values[3]*iq2nl_values[3] + (sumw[i2] - sumw[i1])*iq2nl_values[2]*iq2nl_values[2]
+ + (sumw[i3] - sumw[i2])*iq2nl_values[1]*iq2nl_values[1] + (sumw[kBlockSize] - sumw[i3])*iq2nl_values[0]*iq2nl_values[0];
+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
+ d = sumqx/sumq2; best = d*sumqx; is_shifted = false;
+ }
+ sumqx = (sumx[i1] - sumx[ 0])*shifted_values[3] + (sumx[i2] - sumx[i1])*shifted_values[2]
+ + (sumx[i3] - sumx[i2])*shifted_values[1] + (sumx[kBlockSize] - sumx[i3])*shifted_values[0];
+ sumq2 = (sumw[i1] - sumw[ 0])*shifted_values[3]*shifted_values[3] + (sumw[i2] - sumw[i1])*shifted_values[2]*shifted_values[2]
+ + (sumw[i3] - sumw[i2])*shifted_values[1]*shifted_values[1] + (sumw[kBlockSize] - sumw[i3])*shifted_values[0]*shifted_values[0];
+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
+ d = sumqx/sumq2; best = d*sumqx; is_shifted = true;
+ }
+ }
+ }
+ }
+ scales[ib] = d;
+ if (is_shifted) extra |= (1 << ib);
+
+ float abs_scale = fabsf(scales[ib]);
+ max_abs_scale = MAX(max_abs_scale, abs_scale);
+ }
+
+ if (!max_abs_scale) continue;
+
+ float d = max_abs_scale/15;
+ y[ibl].d = GGML_FP32_TO_FP16(d);
+ y[ibl].extra = extra;
+ float id = 1/d;
+
+ float sumqx = 0, sumq2 = 0;
+ for (int ib = 0; ib < QK_K/kBlockSize; ++ib) {
+ int ls = nearest_int(0.5f*(id*scales[ib]+15));
+ ls = MAX(0, MIN(15, ls));
+ y[ibl].scales[ib/2] |= (ls << 4*(ib%2));
+ ls = 2*ls - 15;
+ float dl = d * ls;
+ if (dl) {
+ const int8_t * block_values = y[ibl].extra & (1 << ib) ? shifted_values : iq2nl_values;
+ const float * xb = xbl + kBlockSize*ib;
+ if (quant_weights) {
+ const float * qw = quant_weights + ibl*QK_K + ib*kBlockSize;
+ for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
+ } else {
+ for (int j = 0; j < kBlockSize; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j];
+ }
+ float idl = 1/dl;
+ int ib32 = ib/2;
+ int offset = 16*(ib%2);
+ uint8_t * qs = y[ibl].qs + 32*(ib32/4) + offset;
+ for (int j = 0; j < 16; ++j) {
+ const float al = idl*xb[j];
+ int ibest = best_index_iq2nl(block_values, al);
+ qs[j] |= (ibest << 2*(ib32%4));
+ float w = weight[j];
+ float q = block_values[ibest]*ls;
+ sumqx += w*q*xb[j];
+ sumq2 += w*q*q;
+ }
+ }
+ }
+ if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(sumqx/sumq2);
+
+ }
+}
+}
+
+void quantize_row_iq2_k_ref(const float * GGML_RESTRICT x, block_iq2_k * GGML_RESTRICT y, int64_t k) {
+ assert(k % QK_K == 0);
+ quantize_iq2_k(x, (void *)y, 1, k, nullptr);
+}
+
+void quantize_row_iq2_k(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
+ assert(k % QK_K == 0);
+ block_iq2_k * y = (block_iq2_k *)vy;
+ quantize_row_iq2_k_ref(x, y, k);
+}
+
+size_t quantize_iq2_k(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+ GGML_ASSERT(n_per_row%QK_K == 0);
+ int nblock = n_per_row/QK_K;
+ char * qrow = (char *)dst;
+ for (int64_t row = 0; row < nrows; ++row) {
+ quantize_row_iq2_k_impl(src, (void *)qrow, n_per_row, imatrix);
+ src += n_per_row;
+ qrow += nblock*sizeof(block_iq2_k);
+ }
+ return nrows * nblock * sizeof(block_iq2_k);
+}
+
+void dequantize_row_iq2_k(const block_iq2_k * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int nb = k / QK_K;
+
+ for (int i = 0; i < nb; i++) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const uint8_t * qs = x[i].qs;
+
+ uint16_t extra = x[i].extra;
+
+ int shift = 0;
+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
+ float dl1 = d * (2*(x[i].scales[ib32] & 0xf) - 15);
+ float dl2 = d * (2*(x[i].scales[ib32] >> 4) - 15);
+ const int8_t * values1 = extra & 1 ? iq2nl_values + 4 : iq2nl_values;
+ const int8_t * values2 = extra & 2 ? iq2nl_values + 4 : iq2nl_values;
+ extra >>= 2;
+ for (int j = 0; j < 16; ++j) {
+ y[j+ 0] = dl1 * values1[(qs[j+ 0] >> shift) & 3];
+ y[j+16] = dl2 * values2[(qs[j+16] >> shift) & 3];
+ }
+ y += 32;
+ shift += 2;
+ if (shift == 8) { qs += 32; shift = 0; }
+ }
+
+ }
+
+}
+
+void vec_dot_iq2_k_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ GGML_UNUSED(nrc);
+ GGML_UNUSED(bx);
+ GGML_UNUSED(by);
+ GGML_UNUSED(bs);
+
+ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_K, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) {
+ return;
+ }
+
+ const int nb = n / QK_K;
+
+ const block_iq2_k * x = (const block_iq2_k *)vx;
+ const block_q8_K * y = (const block_q8_K *)vy;
+}
+
+//
// ============================================== iq4_K
//
void dequantize_row_iq4_k(const block_iq4_k * x, float * y, int64_t k) {
@@ -700,135 +915,297 @@ size_t quantize_iq4_k(const float * src, void * dst, int64_t nrows, int64_t n_pe
}
//
-// ============================================== iq2_K
+// ============================================== iq5_K
//
+void dequantize_row_iq5_k(const block_iq5_k * x, float * y, int64_t k) {
+ assert(k % QK_K == 0);
+ const int nb = k / QK_K;
-namespace {
+ for (int i = 0; i < nb; i++) {
-inline int best_index_iq2nl(const int8_t * values, float x) {
- int idx = x < values[1] ? 0 : x > values[2] ? 2 : 1;
- return x - values[idx] < values[idx+1] - x ? idx : idx + 1;
+ const float d = GGML_FP16_TO_FP32(x[i].d);
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+ const uint8_t * sl = x[i].scales_l;
+ const uint8_t * sh = x[i].scales_h;
+
+ uint16_t extra = x[i].extra;
+
+ int shift = 0;
+ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) {
+
+ float dl1 = d * (((sl[2*ib64+0] & 0xf) | ((sh[ib64] << 4) & 0x30)) - 32);
+ float dl2 = d * (((sl[2*ib64+0] >> 4) | ((sh[ib64] << 2) & 0x30)) - 32);
+ float dl3 = d * (((sl[2*ib64+1] & 0xf) | ((sh[ib64] >> 0) & 0x30)) - 32);
+ float dl4 = d * (((sl[2*ib64+1] >> 4) | ((sh[ib64] >> 2) & 0x30)) - 32);
+ const int8_t * values1 = iq5nl_values + ((extra & 1) << 5);
+ const int8_t * values2 = iq5nl_values + ((extra & 2) << 4);
+ const int8_t * values3 = iq5nl_values + ((extra & 4) << 3);
+ const int8_t * values4 = iq5nl_values + ((extra & 8) << 2);
+ for (int j = 0; j < 16; ++j) {
+ y[j+ 0] = dl1 * values1[(qs[j+ 0] & 0xf) | (((qh[j+ 0] >> shift) & 1) << 4)];
+ y[j+16] = dl2 * values2[(qs[j+16] & 0xf) | (((qh[j+16] >> shift) & 1) << 4)];
+ y[j+32] = dl3 * values3[(qs[j+ 0] >> 4) | (((qh[j+ 0] >> shift) & 2) << 3)];
+ y[j+48] = dl4 * values4[(qs[j+16] >> 4) | (((qh[j+16] >> shift) & 2) << 3)];
+ }
+ y += 64;
+ qs += 32;
+ extra >>= 4;
+ shift += 2;
+ if (shift == 8) { qh += 32; shift = 0; }
+ }
+
+ }
}
-void quantize_row_iq2_k_impl(const float * x, void * vy, int n_per_row, const float * quant_weights) {
+void vec_dot_iq5_k_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc) {
+ assert(n % QK_K == 0);
+ assert(nrc == 1);
+ GGML_UNUSED(nrc);
+ GGML_UNUSED(bx);
+ GGML_UNUSED(by);
+ GGML_UNUSED(bs);
- constexpr int kBlockSize = 16;
+ if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ5_K, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) {
+ return;
+ }
- block_iq2_k * y = (block_iq2_k *)vy;
+ const int nb = n / QK_K;
- float scales[QK_K/kBlockSize];
- float weight[kBlockSize];
- float sumx[kBlockSize+1], sumw[kBlockSize+1];
+ const block_iq5_k * x = (const block_iq5_k *)vx;
+ const block_q8_K * y = (const block_q8_K *)vy;
- std::array<std::pair<float,int>, kBlockSize> pairs;
+ float sumf = 0;
- const int8_t * shifted_values = iq2nl_values + 4;
+ for (int i = 0; i < nb; i++) {
+
+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;
+ const uint8_t * qs = x[i].qs;
+ const uint8_t * qh = x[i].qh;
+ const uint8_t * sl = x[i].scales_l;
+ const uint8_t * sh = x[i].scales_h;
+ const int8_t * q8 = y[i].qs;
+
+ uint16_t extra = x[i].extra;
+
+ int shift = 0;
+ int sumb = 0;
+ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) {
+
+ int dl1 = (((sl[2*ib64+0] & 0xf) | ((sh[ib64] << 4) & 0x30)) - 32);
+ int dl2 = (((sl[2*ib64+0] >> 4) | ((sh[ib64] << 2) & 0x30)) - 32);
+ int dl3 = (((sl[2*ib64+1] & 0xf) | ((sh[ib64] >> 0) & 0x30)) - 32);
+ int dl4 = (((sl[2*ib64+1] >> 4) | ((sh[ib64] >> 2) & 0x30)) - 32);
+ const int8_t * values1 = iq5nl_values + ((extra & 1) << 5);
+ const int8_t * values2 = iq5nl_values + ((extra & 2) << 4);
+ const int8_t * values3 = iq5nl_values + ((extra & 4) << 3);
+ const int8_t * values4 = iq5nl_values + ((extra & 8) << 2);
+ int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0;
+ for (int j = 0; j < 16; ++j) {
+ sumi1 += q8[j+ 0] * values1[(qs[j+ 0] & 0xf) | (((qh[j+ 0] >> shift) & 1) << 4)];
+ sumi2 += q8[j+16] * values2[(qs[j+16] & 0xf) | (((qh[j+16] >> shift) & 1) << 4)];
+ sumi3 += q8[j+32] * values3[(qs[j+ 0] >> 4) | (((qh[j+ 0] >> shift) & 2) << 3)];
+ sumi4 += q8[j+48] * values4[(qs[j+16] >> 4) | (((qh[j+16] >> shift) & 2) << 3)];
+ }
+ sumb += dl1 * sumi1 + dl2 * sumi2 + dl3 * sumi3 + dl4 * sumi4;
+ q8 += 64;
+ qs += 32;
+ extra >>= 4;
+ shift += 2;
+ }
+ sumf += d * sumb;
+
+ }
+
+ *s = sumf;
+
+}
+
+namespace {
+static int8_t iq5nl_index[248] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6,
+ 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10,
+ 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 16, 16, 16,
+ 16, 16, 16, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21,
+ 21, 21, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25,
+ 25, 25, 26, 26, 26, 26, 26, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 29,
+ 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30
+};
+static inline int best_index_iq5nl(const int8_t * values, float x) {
+ if (x <= values[ 0]) return 0;
+ if (x >= values[31]) return 31;
+ int index = iq5nl_index[(int)x - values[0]];
+ return x - values[index] < values[index+1] - x ? index : index+1;
+}
+
+void quantize_row_iq5_k_impl(const float * x, void * vy, int n_per_row, const float * quant_weights) {
+ const int ntry = 5;
+ const float step = 1.f;
+
+ block_iq5_k * y = (block_iq5_k *)vy;
+
+ float scales[QK_K/16];
+ float weight[16];
+
+ const int8_t * shifted_values = iq5nl_values + 32;
for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) {
- memset(&y[ibl], 0, sizeof(block_iq2_k));
+ memset(&y[ibl], 0, sizeof(block_iq5_k));
y[ibl].d = GGML_FP32_TO_FP16(0.f);
const float * xbl = x + ibl*QK_K;
float sumx2 = 0;
for (int j = 0; j < QK_K; ++j) sumx2 += xbl[j]*xbl[j];
- const float sigma2 = 1.5f*sumx2/QK_K;
+ const float sigma2 = 2*sumx2/QK_K;
+ float max_scale = 0, max_abs_scale = 0;
uint16_t extra = 0;
- float max_abs_scale = 0;
-
- for (int ib = 0; ib < QK_K/kBlockSize; ++ib) {
- const float * xb = xbl + kBlockSize*ib;
+ for (int ib = 0; ib < QK_K/16; ++ib) {
+ const float * xb = xbl + 16*ib;
if (quant_weights) {
- const float * qw = quant_weights + ibl*QK_K + ib*kBlockSize;
- for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
+ const float * qw = quant_weights + ibl*QK_K + ib*16;
+ for (int j = 0; j < 16; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
} else {
- for (int j = 0; j < kBlockSize; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j];
+ for (int j = 0; j < 16; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j];
}
- for (int j = 0; j < kBlockSize; ++j) pairs[j] = {xb[j], j};
- std::sort(pairs.begin(), pairs.end());
- sumx[0] = sumw[0] = 0;
- for (int j = 0; j < kBlockSize; ++j) {
- int jj = pairs[j].second;
- sumw[j+1] = sumw[j] + weight[jj];
- sumx[j+1] = sumx[j] + weight[jj]*xb[jj];
+ float amax = 0, max = 0;
+ for (int j = 0; j < 16; ++j) {
+ float ax = fabsf(xb[j]);
+ if (ax > amax) {
+ amax = ax; max = xb[j];
+ }
+ }
+ if (!amax) {
+ scales[ib] = 0;
+ continue;
+ }
+ float d = ntry > 0 ? -max/iq5nl_values[0] : max/iq5nl_values[0];
+ float id = 1/d;
+ float sumqx_p = 0, sumq2_p = 0;
+ float sumqx_m = 0, sumq2_m = 0;
+ for (int j = 0; j < 16; ++j) {
+ float w = weight[j];
+ float al = id*xb[j];
+ int l = best_index_iq5nl(iq5nl_values, al);
+ float q = iq5nl_values[l];
+ sumqx_p += w*q*xb[j];
+ sumq2_p += w*q*q;
+ l = best_index_iq5nl(iq5nl_values, -al);
+ q = iq5nl_values[l];
+ sumqx_m += w*q*xb[j];
+ sumq2_m += w*q*q;
+ }
+ d = sumqx_p/sumq2_p;
+ float best = d*sumqx_p;
+ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) {
+ d = sumqx_m/sumq2_m; best = d*sumqx_m;
}
- float best = 0, d = 0;
bool is_shifted = false;
- float sumqx, sumq2;
- for (int i1 = 0; i1 < kBlockSize; ++i1) {
- for (int i2 = i1; i2 < kBlockSize; ++i2) {
- for (int i3 = i2; i3 < kBlockSize; ++i3) {
- sumqx = (sumx[i1] - sumx[ 0])*iq2nl_values[0] + (sumx[i2] - sumx[i1])*iq2nl_values[1]
- + (sumx[i3] - sumx[i2])*iq2nl_values[2] + (sumx[kBlockSize] - sumx[i3])*iq2nl_values[3];
- sumq2 = (sumw[i1] - sumw[ 0])*iq2nl_values[0]*iq2nl_values[0] + (sumw[i2] - sumw[i1])*iq2nl_values[1]*iq2nl_values[1]
- + (sumw[i3] - sumw[i2])*iq2nl_values[2]*iq2nl_values[2] + (sumw[kBlockSize] - sumw[i3])*iq2nl_values[3]*iq2nl_values[3];
- if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
- d = sumqx/sumq2; best = d*sumqx; is_shifted = false;
- }
- sumqx = (sumx[i1] - sumx[ 0])*shifted_values[0] + (sumx[i2] - sumx[i1])*shifted_values[1]
- + (sumx[i3] - sumx[i2])*shifted_values[2] + (sumx[kBlockSize] - sumx[i3])*shifted_values[3];
- sumq2 = (sumw[i1] - sumw[ 0])*shifted_values[0]*shifted_values[0] + (sumw[i2] - sumw[i1])*shifted_values[1]*shifted_values[1]
- + (sumw[i3] - sumw[i2])*shifted_values[2]*shifted_values[2] + (sumw[kBlockSize] - sumw[i3])*shifted_values[3]*shifted_values[3];
- if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
- d = sumqx/sumq2; best = d*sumqx; is_shifted = true;
- }
- sumqx = (sumx[i1] - sumx[ 0])*iq2nl_values[3] + (sumx[i2] - sumx[i1])*iq2nl_values[2]
- + (sumx[i3] - sumx[i2])*iq2nl_values[1] + (sumx[kBlockSize] - sumx[i3])*iq2nl_values[0];
- sumq2 = (sumw[i1] - sumw[ 0])*iq2nl_values[3]*iq2nl_values[3] + (sumw[i2] - sumw[i1])*iq2nl_values[2]*iq2nl_values[2]
- + (sumw[i3] - sumw[i2])*iq2nl_values[1]*iq2nl_values[1] + (sumw[kBlockSize] - sumw[i3])*iq2nl_values[0]*iq2nl_values[0];
- if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
- d = sumqx/sumq2; best = d*sumqx; is_shifted = false;
- }
- sumqx = (sumx[i1] - sumx[ 0])*shifted_values[3] + (sumx[i2] - sumx[i1])*shifted_values[2]
- + (sumx[i3] - sumx[i2])*shifted_values[1] + (sumx[kBlockSize] - sumx[i3])*shifted_values[0];
- sumq2 = (sumw[i1] - sumw[ 0])*shifted_values[3]*shifted_values[3] + (sumw[i2] - sumw[i1])*shifted_values[2]*shifted_values[2]
- + (sumw[i3] - sumw[i2])*shifted_values[1]*shifted_values[1] + (sumw[kBlockSize] - sumw[i3])*shifted_values[0]*shifted_values[0];
- if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {
- d = sumqx/sumq2; best = d*sumqx; is_shifted = true;
- }
- }
+ for (int itry = -ntry; itry <= ntry; ++itry) {
+ id = (itry*step + iq5nl_values[0])/max;
+ sumqx_p = sumq2_p = 0;
+ sumqx_m = sumq2_m = 0;
+ for (int j = 0; j < 16; ++j) {
+ float w = weight[j];
+ float al = id*xb[j];
+ int l = best_index_iq5nl(iq5nl_values, al);
+ float q = iq5nl_values[l];
+ sumqx_p += w*q*xb[j];
+ sumq2_p += w*q*q;
+ l = best_index_iq5nl(iq5nl_values, -al);
+ q = iq5nl_values[l];
+ sumqx_m += w*q*xb[j];
+ sumq2_m += w*q*q;
+ }
+ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) {
+ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = false;
+ }
+ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) {
+ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = false;
+ }
+ id = (itry*step + shifted_values[0])/max;
+ sumqx_p = sumq2_p = 0;
+ sumqx_m = sumq2_m = 0;
+ for (int j = 0; j < 16; ++j) {
+ float w = weight[j];
+ float al = id*xb[j];
+ int l = best_index_iq5nl(shifted_values, al);
+ float q = shifted_values[l];
+ sumqx_p += w*q*xb[j];
+ sumq2_p += w*q*q;
+ l = best_index_iq5nl(shifted_values, -al);
+ q = shifted_values[l];
+ sumqx_m += w*q*xb[j];
+ sumq2_m += w*q*q;
+ }
+ if (sumq2_p > 0 && sumqx_p*sumqx_p > best*sumq2_p) {
+ d = sumqx_p/sumq2_p; best = d * sumqx_p; is_shifted = true;
+ }
+ if (sumq2_m > 0 && sumqx_m*sumqx_m > best*sumq2_m) {
+ d = sumqx_m/sumq2_m; best = d * sumqx_m; is_shifted = true;
}
}
+ if (d) {
+ const int8_t * block_values = is_shifted ? shifted_values : iq5nl_values;
+ float sumqx = 0, sumq2 = 0;
+ id = 1/d;
+ for (int j = 0; j < 16; ++j) {
+ float w = weight[j];
+ float al = id*xb[j];
+ int l = best_index_iq5nl(block_values, al);
+ float q = block_values[l];
+ sumqx += w*q*xb[j];
+ sumq2 += w*q*q;
+ }
+ if (sumq2 > 0) d = sumqx/sumq2;
+ }
scales[ib] = d;
if (is_shifted) extra |= (1 << ib);
float abs_scale = fabsf(scales[ib]);
- max_abs_scale = MAX(max_abs_scale, abs_scale);
+ if (abs_scale > max_abs_scale) {
+ max_abs_scale = abs_scale; max_scale = scales[ib];
+ }
+
}
if (!max_abs_scale) continue;
-
- float d = max_abs_scale/15;
+ float d = -max_scale/32;
y[ibl].d = GGML_FP32_TO_FP16(d);
y[ibl].extra = extra;
+
float id = 1/d;
float sumqx = 0, sumq2 = 0;
- for (int ib = 0; ib < QK_K/kBlockSize; ++ib) {
- int ls = nearest_int(0.5f*(id*scales[ib]+15));
- ls = MAX(0, MIN(15, ls));
- y[ibl].scales[ib/2] |= (ls << 4*(ib%2));
- ls = 2*ls - 15;
+ for (int ib = 0; ib < QK_K/16; ++ib) {
+ int ls = nearest_int(id*scales[ib]);
+ ls = MAX(-32, MIN(31, ls));
+ int uls = ls + 32;
+ y[ibl].scales_l[ib/2] |= ((uls & 0xf) << 4*(ib%2));
+ y[ibl].scales_h[ib/4] |= ((uls >> 4) << 2*(ib%4));
float dl = d * ls;
if (dl) {
- const int8_t * block_values = y[ibl].extra & (1 << ib) ? shifted_values : iq2nl_values;
- const float * xb = xbl + kBlockSize*ib;
+ const int8_t * block_values = y[ibl].extra & (1 << ib) ? shifted_values : iq5nl_values;
+ const float * xb = xbl + 16*ib;
if (quant_weights) {
- const float * qw = quant_weights + ibl*QK_K + ib*kBlockSize;
- for (int j = 0; j < kBlockSize; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
+ const float * qw = quant_weights + ibl*QK_K + ib*16;
+ for (int j = 0; j < 16; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
} else {
- for (int j = 0; j < kBlockSize; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j];
+ for (int j = 0; j < 16; ++j) weight[j] = 0.25f*sigma2 + xb[j]*xb[j];
}
float idl = 1/dl;
int ib32 = ib/2;
int offset = 16*(ib%2);
- uint8_t * qs = y[ibl].qs + 32*(ib32/4) + offset;
+ uint8_t * qs = y[ibl].qs + 32*(ib32/2) + offset;
+ uint8_t * qh = y[ibl].qh + 32*(ib32/8) + offset;
for (int j = 0; j < 16; ++j) {
const float al = idl*xb[j];
- int ibest = best_index_iq2nl(block_values, al);
- qs[j] |= (ibest << 2*(ib32%4));
+ int ibest = best_index_iq5nl(block_values, al);
+ qs[j] |= ((ibest & 0xf) << 4*(ib32%2));
+ qh[j] |= ((ibest >> 4) << (ib32%8));
float w = weight[j];
float q = block_values[ibest]*ls;
sumqx += w*q*xb[j];
@@ -839,77 +1216,30 @@ void quantize_row_iq2_k_impl(const float * x, void * vy, int n_per_row, const fl
if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(sumqx/sumq2);
}
+
}
+
}
-void quantize_row_iq2_k_ref(const float * GGML_RESTRICT x, block_iq2_k * GGML_RESTRICT y, int64_t k) {
+void quantize_row_iq5_k_ref(const float * x, block_iq5_k * y, int64_t k) {
assert(k % QK_K == 0);
- quantize_iq2_k(x, (void *)y, 1, k, nullptr);
+ quantize_iq5_k(x, (void *)y, 1, k, nullptr);
}
-void quantize_row_iq2_k(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
+void quantize_row_iq5_k(const float * x, void * vy, int64_t k) {
assert(k % QK_K == 0);
- block_iq2_k * y = (block_iq2_k *)vy;
- quantize_row_iq2_k_ref(x, y, k);
+ block_iq5_k * y = (block_iq5_k *)vy;
+ quantize_row_iq5_k_ref(x, y, k);
}
-size_t quantize_iq2_k(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
+size_t quantize_iq5_k(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix) {
GGML_ASSERT(n_per_row%QK_K == 0);
int nblock = n_per_row/QK_K;
char * qrow = (char *)dst;
for (int64_t row = 0; row < nrows; ++row) {
- quantize_row_iq2_k_impl(src, (void *)qrow, n_per_row, imatrix);
+ quantize_row_iq5_k_impl(src, (void *)qrow, n_per_row, imatrix);
src += n_per_row;
- qrow += nblock*sizeof(block_iq2_k);
- }
- return nrows * nblock * sizeof(block_iq2_k);
-}
-
-void dequantize_row_iq2_k(const block_iq2_k * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
- assert(k % QK_K == 0);
- const int nb = k / QK_K;
-
- for (int i = 0; i < nb; i++) {
-
- const float d = GGML_FP16_TO_FP32(x[i].d);
- const uint8_t * qs = x[i].qs;
-
- uint16_t extra = x[i].extra;
-
- int shift = 0;
- for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
- float dl1 = d * (2*(x[i].scales[ib32] & 0xf) - 15);
- float dl2 = d * (2*(x[i].scales[ib32] >> 4) - 15);
- const int8_t * values1 = extra & 1 ? iq2nl_values + 4 : iq2nl_values;
- const int8_t * values2 = extra & 2 ? iq2nl_values + 4 : iq2nl_values;
- extra >>= 2;
- for (int j = 0; j < 16; ++j) {
- y[j+ 0] = dl1 * values1[(qs[j+ 0] >> shift) & 3];
- y[j+16] = dl2 * values2[(qs[j+16] >> shift) & 3];
- }
- y += 32;
- shift += 2;
- if (shift == 8) { qs += 32; shift = 0; }
- }
-
+ qrow += nblock*sizeof(block_iq5_k);
}
-
-}
-
-void vec_dot_iq2_k_q8_k(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
- assert(n % QK_K == 0);
- assert(nrc == 1);
- GGML_UNUSED(nrc);
- GGML_UNUSED(bx);
- GGML_UNUSED(by);
- GGML_UNUSED(bs);
-
- if (iqk_mul_mat(1, 1, n, GGML_TYPE_IQ2_K, vx, 0, GGML_TYPE_Q8_K, vy, 0, s, 0, 0, 1)) {
- return;
- }
-
- const int nb = n / QK_K;
-
- const block_iq2_k * x = (const block_iq2_k *)vx;
- const block_q8_K * y = (const block_q8_K *)vy;
+ return nrows * nblock * sizeof(block_iq5_k);
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-23 18:43:13 +0000