diff options
| author | Iwan Kawrakow <iwan.kawrakow@gmail.com> | 2024-06-19 16:46:23 +0300 |
|---|---|---|
| committer | Iwan Kawrakow <iwan.kawrakow@gmail.com> | 2024-06-22 12:02:52 +0300 |
| commit | 58d9e8f1d2efba4b6717043f7a5167be670a6f2e (patch) | |
| tree | bc70f7b1197e9572c3efdfa84d349b729c41cf9b /ggml-common.h | |
| parent | 927e251a12fa287e13c6bd9667ee97d783486c09 (diff) | |
bitnet: put the scale in a separate tensor
and correspondingly add an extra ggml_mul_mat operation.
As per @ggerganov, this is how things should be done.
It seems to be working, but as far as I can tell this
results in a ~15% performance penalty for prompt processing.
Commiting so I can go and test on othe platforms.
Diffstat (limited to 'ggml-common.h')
| -rw-r--r-- | ggml-common.h | 7 |
1 files changed, 3 insertions, 4 deletions
diff --git a/ggml-common.h b/ggml-common.h index f5a35960..d3945975 100644 --- a/ggml-common.h +++ b/ggml-common.h @@ -375,20 +375,19 @@ static_assert(sizeof(block_iq1_m) == QK_K/8 + QK_K/16 + QK_K/32, "wrong iq1_m bl // #define QK_IQ1BN 64 typedef struct { - uint16_t extra; + uint8_t extra; uint8_t ql[QK_IQ1BN/8]; uint8_t qh[QK_IQ1BN/16]; } block_iq1_bn; -static_assert(sizeof(block_iq1_bn) == sizeof(uint16_t) + QK_IQ1BN/8 + QK_IQ1BN/16, "wrong iq1_bn block size/padding"); +static_assert(sizeof(block_iq1_bn) == sizeof(uint8_t) + QK_IQ1BN/8 + QK_IQ1BN/16, "wrong iq1_bn block size/padding"); // // Bitnet - implemented as 2.25 bpw // #define QK_IQ2BN 64 typedef struct { - ggml_half d; uint8_t qs[QK_IQ2BN/4]; } block_iq2_bn; -static_assert(sizeof(block_iq2_bn) == sizeof(ggml_half) + QK_IQ2BN/4, "wrong iq2_bn block size/padding"); +static_assert(sizeof(block_iq2_bn) == QK_IQ2BN/4, "wrong iq2_bn block size/padding"); // Used by IQ1_M quants typedef union { |