summaryrefslogtreecommitdiff
path: root/gguf-py
diff options
context:
space:
mode:
Diffstat (limited to 'gguf-py')
-rw-r--r--gguf-py/gguf/constants.py344
1 files changed, 227 insertions, 117 deletions
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 22cde145..6e03f9a4 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -1171,47 +1171,86 @@ class PoolingType(IntEnum):
class GGMLQuantizationType(IntEnum):
- F32 = 0
- F16 = 1
- Q4_0 = 2
- Q4_1 = 3
- Q5_0 = 6
- Q5_1 = 7
- Q8_0 = 8
- Q8_1 = 9
- Q2_K = 10
- Q3_K = 11
- Q4_K = 12
- Q5_K = 13
- Q6_K = 14
- Q8_K = 15
- IQ2_XXS = 16
- IQ2_XS = 17
- IQ3_XXS = 18
- IQ1_S = 19
- IQ4_NL = 20
- IQ3_S = 21
- IQ2_S = 22
- IQ4_XS = 23
- I8 = 24
- I16 = 25
- I32 = 26
- I64 = 27
- F64 = 28
- IQ1_M = 29
- BF16 = 30
- Q4_0_4_4 = 31
- Q4_0_4_8 = 32
- Q4_0_8_8 = 33
- IQ1_BN = 34,
- IQ2_BN = 35,
- Q8_K64 = 36,
- IQ2_K = 37,
- IQ3_K = 38,
- IQ4_K = 39,
- IQ5_K = 40,
- IQ6_K = 41,
- IQ2_TN = 42,
+ F32 = 0
+ F16 = 1
+ Q4_0 = 2
+ Q4_1 = 3
+ Q5_0 = 6
+ Q5_1 = 7
+ Q8_0 = 8
+ Q8_1 = 9
+ Q2_K = 10
+ Q3_K = 11
+ Q4_K = 12
+ Q5_K = 13
+ Q6_K = 14
+ Q8_K = 15
+ IQ2_XXS = 16
+ IQ2_XS = 17
+ IQ3_XXS = 18
+ IQ1_S = 19
+ IQ4_NL = 20
+ IQ3_S = 21
+ IQ2_S = 22
+ IQ4_XS = 23
+ I8 = 24
+ I16 = 25
+ I32 = 26
+ I64 = 27
+ F64 = 28
+ IQ1_M = 29
+ BF16 = 30
+ Q4_0_4_4 = 31
+ Q4_0_4_8 = 32
+ Q4_0_8_8 = 33
+ I2_S = 36
+ Q8_0_X4 = 97
+ Q8_1_X4 = 98
+ Q8_2_X4 = 99
+ Q6_0 = 133
+ IQ1_BN = 134
+ IQ2_BN = 135
+ Q8_K64 = 136
+ IQ2_K = 137
+ IQ3_K = 138
+ IQ4_K = 139
+ IQ5_K = 140
+ IQ6_K = 141
+ IQ4_KS = 144
+ IQ2_KS = 145
+ IQ4_KSS = 146
+ Q8_K16 = 147
+ Q8_K32 = 148
+ Q8_KR8 = 149
+ Q8_K128 = 150
+ Q8_KV = 151
+ Q4_0_R8 = 202
+ Q5_0_R4 = 206
+ Q8_0_R8 = 208
+ Q2_K_R4 = 210
+ Q3_K_R4 = 211
+ Q4_K_R4 = 212
+ Q5_K_R4 = 213
+ Q6_K_R4 = 214
+ IQ2_XXS_R4= 216
+ IQ2_XS_R4 = 217
+ IQ3_XXS_R4= 218
+ IQ1_S_R4 = 219
+ IQ4_NL_R4 = 220
+ IQ3_S_R4 = 221
+ IQ2_S_R4 = 222
+ IQ4_XS_R8 = 223
+ IQ1_M_R4 = 229
+ BF16_R16 = 230
+ Q6_0_R4 = 233
+ IQ2_BN_R4 = 335
+ IQ2_K_R4 = 337
+ IQ3_K_R4 = 338
+ IQ4_K_R4 = 339
+ IQ5_K_R4 = 340
+ IQ4_KS_R4 = 344
+ Q8_KV_R8 = 398
+ Q8_K_R8 = 399
class ExpertGatingFuncType(IntEnum):
@@ -1225,50 +1264,71 @@ class ExpertGatingFuncType(IntEnum):
# from llama_ftype in llama.h
# ALL VALUES SHOULD BE THE SAME HERE AS THEY ARE OVER THERE.
class LlamaFileType(IntEnum):
- ALL_F32 = 0
- MOSTLY_F16 = 1 # except 1d tensors
- MOSTLY_Q4_0 = 2 # except 1d tensors
- MOSTLY_Q4_1 = 3 # except 1d tensors
- # MOSTLY_Q4_1_SOME_F16 = 4 # tok_embeddings.weight and output.weight are F16
- # MOSTLY_Q4_2 = 5 # support has been removed
- # MOSTLY_Q4_3 = 6 # support has been removed
- MOSTLY_Q8_0 = 7 # except 1d tensors
- MOSTLY_Q5_0 = 8 # except 1d tensors
- MOSTLY_Q5_1 = 9 # except 1d tensors
- MOSTLY_Q2_K = 10 # except 1d tensors
- MOSTLY_Q3_K_S = 11 # except 1d tensors
- MOSTLY_Q3_K_M = 12 # except 1d tensors
- MOSTLY_Q3_K_L = 13 # except 1d tensors
- MOSTLY_Q4_K_S = 14 # except 1d tensors
- MOSTLY_Q4_K_M = 15 # except 1d tensors
- MOSTLY_Q5_K_S = 16 # except 1d tensors
- MOSTLY_Q5_K_M = 17 # except 1d tensors
- MOSTLY_Q6_K = 18 # except 1d tensors
- MOSTLY_IQ2_XXS = 19 # except 1d tensors
- MOSTLY_IQ2_XS = 20 # except 1d tensors
- MOSTLY_Q2_K_S = 21 # except 1d tensors
- MOSTLY_IQ3_XS = 22 # except 1d tensors
- MOSTLY_IQ3_XXS = 23 # except 1d tensors
- MOSTLY_IQ1_S = 24 # except 1d tensors
- MOSTLY_IQ4_NL = 25 # except 1d tensors
- MOSTLY_IQ3_S = 26 # except 1d tensors
- MOSTLY_IQ3_M = 27 # except 1d tensors
- MOSTLY_IQ2_S = 28 # except 1d tensors
- MOSTLY_IQ2_M = 29 # except 1d tensors
- MOSTLY_IQ4_XS = 30 # except 1d tensors
- MOSTLY_IQ1_M = 31 # except 1d tensors
- MOSTLY_BF16 = 32 # except 1d tensors
- MOSTLY_Q4_0_4_4 = 33 # except 1d tensors
- MOSTLY_Q4_0_4_8 = 34 # except 1d tensors
- MOSTLY_Q4_0_8_8 = 35 # except 1d tensors
- MOSTLY_IQ1_BN = 36, # except 1d tensors
- MOSTLY_IQ2_BN = 37, # except 1d tensors
- MOSTLY_IQ2_K = 38, # except 1d tensors
- MOSTLY_IQ3_K = 39, # except 1d tensors
- MOSTLY_IQ4_K = 40, # except 1d tensors
- MOSTLY_IQ5_K = 41, # except 1d tensors
- MOSTLY_IQ6_K = 42, # except 1d tensors
- MOSTLY_IQ2_TN = 43, # except 1d tensors
+ ALL_F32 = 0
+ MOSTLY_F16 = 1 #except 1d tensors
+ MOSTLY_Q4_0 = 2 #except 1d tensors
+ MOSTLY_Q4_1 = 3 #except 1d tensors
+ MOSTLY_Q4_1_SOME_F16 = 4 #tok_embeddings.weight and output.weight are F16
+ MOSTLY_Q8_0 = 7 #except 1d tensors
+ MOSTLY_Q5_0 = 8 #except 1d tensors
+ MOSTLY_Q5_1 = 9 #except 1d tensors
+ MOSTLY_Q2_K = 10 #except 1d tensors
+ MOSTLY_Q3_K = 11 #except 1d tensors
+ MOSTLY_Q4_K = 12 #except 1d tensors
+ MOSTLY_Q5_K = 13 #except 1d tensors
+ MOSTLY_Q6_K = 14 #except 1d tensors
+ MOSTLY_IQ2_XXS = 15 #except 1d tensors
+ MOSTLY_IQ2_XS = 16 #except 1d tensors
+ MOSTLY_IQ3_XXS = 17 #except 1d tensors
+ MOSTLY_IQ1_S = 18 #except 1d tensors
+ MOSTLY_IQ4_NL = 19 #except 1d tensors
+ MOSTLY_IQ3_S = 20 #except 1d tensors
+ MOSTLY_IQ2_S = 21 #except 1d tensors
+ MOSTLY_IQ4_XS = 22 #except 1d tensors
+ MOSTLY_IQ1_M = 23 #except 1d tensors
+ MOSTLY_BF16 = 24 #except 1d tensors
+ MOSTLY_Q4_0_4_4 = 25 #except 1d tensors
+ MOSTLY_Q4_0_4_8 = 26 #except 1d tensors
+ MOSTLY_Q4_0_8_8 = 27 #except 1d tensors
+ MOSTLY_Q6_0 = 127 #except 1d tensors
+ MOSTLY_IQ1_BN = 128 #except 1d tensors
+ MOSTLY_IQ2_BN = 129 #except 1d tensors
+ MOSTLY_IQ2_K = 130 #except 1d tensors
+ MOSTLY_IQ3_K = 131 #except 1d tensors
+ MOSTLY_IQ4_K = 132 #except 1d tensors
+ MOSTLY_IQ5_K = 133 #except 1d tensors
+ MOSTLY_IQ6_K = 134 #except 1d tensors
+ MOSTLY_IQ4_KS = 137 #except 1d tensors
+ MOSTLY_IQ2_KS = 138 #except 1d tensors
+ MOSTLY_IQ4_KSS = 139 #except 1d tensors
+ MOSTLY_Q8_KV = 140 #except 1d tensors
+ MOSTLY_Q4_0_R8 = 202 #except 1d tensors
+ MOSTLY_Q8_0_R8 = 207 #except 1d tensors
+ MOSTLY_Q5_0_R4 = 208 #except 1d tensors
+ MOSTLY_Q2_K_R4 = 210 #except 1d tensors
+ MOSTLY_Q3_K_R4 = 211 #except 1d tensors
+ MOSTLY_Q4_K_R4 = 212 #except 1d tensors
+ MOSTLY_Q5_K_R4 = 213 #except 1d tensors
+ MOSTLY_Q6_K_R4 = 214 #except 1d tensors
+ MOSTLY_IQ2_XXS_R4 = 215 #except 1d tensors
+ MOSTLY_IQ2_XS_R4 = 216 #except 1d tensors
+ MOSTLY_IQ3_XXS_R4 = 217 #except 1d tensors
+ MOSTLY_IQ1_S_R4 = 218 #except 1d tensors
+ MOSTLY_IQ4_NL_R4 = 219 #except 1d tensors
+ MOSTLY_IQ3_S_R4 = 220 #except 1d tensors
+ MOSTLY_IQ2_S_R4 = 221 #except 1d tensors
+ MOSTLY_IQ4_XS_R8 = 222 #except 1d tensors
+ MOSTLY_IQ1_M_R4 = 223 #except 1d tensors
+ MOSTLY_BF16_R16 = 224 #except 1d tensors
+ MOSTLY_Q6_0_R4 = 227 #except 1d tensors
+ MOSTLY_IQ2_BN_R4 = 329 #except 1d tensors
+ MOSTLY_IQ2_K_R4 = 330 #except 1d tensors
+ MOSTLY_IQ3_K_R4 = 331 #except 1d tensors
+ MOSTLY_IQ4_K_R4 = 332 #except 1d tensors
+ MOSTLY_IQ5_K_R4 = 333 #except 1d tensors
+ MOSTLY_IQ4_KS_R4 = 337 #except 1d tensors
+ MOSTLY_Q8_KV_R8 = 398 #except 1d tensors
+ MOSTLY_Q8_K_R8 = 399 #except 1d tensors
GUESSED = 1024 # not specified in the model file
@@ -1313,39 +1373,89 @@ class GGUFValueType(IntEnum):
# Items here are (block size, type size)
QK_K = 256
+
+#Values generated programatically
GGML_QUANT_SIZES: dict[GGMLQuantizationType, tuple[int, int]] = {
- GGMLQuantizationType.F32: (1, 4),
- GGMLQuantizationType.F16: (1, 2),
- GGMLQuantizationType.Q4_0: (32, 2 + 16),
- GGMLQuantizationType.Q4_1: (32, 2 + 2 + 16),
- GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16),
- GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16),
- GGMLQuantizationType.Q8_0: (32, 2 + 32),
- GGMLQuantizationType.Q8_1: (32, 4 + 4 + 32),
- GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4),
- GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12),
- GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12),
- GGMLQuantizationType.Q5_K: (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),
- GGMLQuantizationType.Q6_K: (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),
- GGMLQuantizationType.Q8_K: (256, 4 + QK_K + QK_K // 8),
- GGMLQuantizationType.IQ2_XXS: (256, 2 + QK_K // 4),
- GGMLQuantizationType.IQ2_XS: (256, 2 + QK_K // 4 + QK_K // 32),
- GGMLQuantizationType.IQ3_XXS: (256, 2 + QK_K // 4 + QK_K // 8),
- GGMLQuantizationType.IQ1_S: (256, 2 + QK_K // 8 + QK_K // 16),
- GGMLQuantizationType.IQ4_NL: (32, 2 + 16),
- GGMLQuantizationType.IQ3_S: (256, 2 + QK_K // 4 + QK_K // 8 + QK_K // 32 + 4),
- GGMLQuantizationType.IQ2_S: (256, 2 + QK_K // 4 + QK_K // 16),
- GGMLQuantizationType.IQ4_XS: (256, 2 + 2 + QK_K // 2 + QK_K // 64),
- GGMLQuantizationType.I8: (1, 1),
- GGMLQuantizationType.I16: (1, 2),
- GGMLQuantizationType.I32: (1, 4),
- GGMLQuantizationType.I64: (1, 8),
- GGMLQuantizationType.F64: (1, 8),
- GGMLQuantizationType.IQ1_M: (256, QK_K // 8 + QK_K // 16 + QK_K // 32),
- GGMLQuantizationType.BF16: (1, 2),
- GGMLQuantizationType.Q4_0_4_4:(32, 2 + 16),
- GGMLQuantizationType.Q4_0_4_8:(32, 2 + 16),
- GGMLQuantizationType.Q4_0_8_8:(32, 2 + 16),
+ GGMLQuantizationType.F32 : ( 1, 4),
+ GGMLQuantizationType.F16 : ( 1, 2),
+ GGMLQuantizationType.Q4_0 : ( 32, 18),
+ GGMLQuantizationType.Q4_1 : ( 32, 20),
+ GGMLQuantizationType.Q5_0 : ( 32, 22),
+ GGMLQuantizationType.Q5_1 : ( 32, 24),
+ GGMLQuantizationType.Q8_0 : ( 32, 34),
+ GGMLQuantizationType.Q8_1 : ( 32, 36),
+ GGMLQuantizationType.Q2_K : ( 256, 84),
+ GGMLQuantizationType.Q3_K : ( 256, 110),
+ GGMLQuantizationType.Q4_K : ( 256, 144),
+ GGMLQuantizationType.Q5_K : ( 256, 176),
+ GGMLQuantizationType.Q6_K : ( 256, 210),
+ GGMLQuantizationType.Q8_K : ( 256, 292),
+ GGMLQuantizationType.IQ2_XXS : ( 256, 66),
+ GGMLQuantizationType.IQ2_XS : ( 256, 74),
+ GGMLQuantizationType.IQ3_XXS : ( 256, 98),
+ GGMLQuantizationType.IQ1_S : ( 256, 50),
+ GGMLQuantizationType.IQ4_NL : ( 32, 18),
+ GGMLQuantizationType.IQ3_S : ( 256, 110),
+ GGMLQuantizationType.IQ2_S : ( 256, 82),
+ GGMLQuantizationType.IQ4_XS : ( 256, 136),
+ GGMLQuantizationType.I8 : ( 1, 1),
+ GGMLQuantizationType.I16 : ( 1, 2),
+ GGMLQuantizationType.I32 : ( 1, 4),
+ GGMLQuantizationType.I64 : ( 1, 8),
+ GGMLQuantizationType.F64 : ( 1, 8),
+ GGMLQuantizationType.IQ1_M : ( 256, 56),
+ GGMLQuantizationType.BF16 : ( 1, 2),
+ GGMLQuantizationType.Q4_0_4_4 : ( 32, 18),
+ GGMLQuantizationType.Q4_0_4_8 : ( 32, 18),
+ GGMLQuantizationType.Q4_0_8_8 : ( 32, 18),
+ GGMLQuantizationType.I2_S : ( 1, 1),
+ GGMLQuantizationType.Q8_0_X4 : ( 32, 34),
+ GGMLQuantizationType.Q8_1_X4 : ( 32, 36),
+ GGMLQuantizationType.Q8_2_X4 : ( 32, 36),
+ GGMLQuantizationType.Q6_0 : ( 32, 26),
+ GGMLQuantizationType.IQ1_BN : ( 64, 13),
+ GGMLQuantizationType.IQ2_BN : ( 64, 16),
+ GGMLQuantizationType.Q8_K64 : ( 64, 68),
+ GGMLQuantizationType.IQ2_K : ( 256, 76),
+ GGMLQuantizationType.IQ3_K : ( 256, 110),
+ GGMLQuantizationType.IQ4_K : ( 256, 144),
+ GGMLQuantizationType.IQ5_K : ( 256, 176),
+ GGMLQuantizationType.IQ6_K : ( 256, 212),
+ GGMLQuantizationType.IQ4_KS : ( 256, 136),
+ GGMLQuantizationType.IQ2_KS : ( 256, 70),
+ GGMLQuantizationType.IQ4_KSS : ( 256, 128),
+ GGMLQuantizationType.Q8_K16 : ( 64, 64),
+ GGMLQuantizationType.Q8_K32 : ( 256, 292),
+ GGMLQuantizationType.Q8_KR8 : ( 256, 292),
+ GGMLQuantizationType.Q8_K128 : ( 128, 140),
+ GGMLQuantizationType.Q8_KV : ( 32, 32),
+ GGMLQuantizationType.Q4_0_R8 : ( 32, 18),
+ GGMLQuantizationType.Q5_0_R4 : ( 32, 22),
+ GGMLQuantizationType.Q8_0_R8 : ( 32, 34),
+ GGMLQuantizationType.Q2_K_R4 : ( 256, 84),
+ GGMLQuantizationType.Q3_K_R4 : ( 256, 110),
+ GGMLQuantizationType.Q4_K_R4 : ( 256, 144),
+ GGMLQuantizationType.Q5_K_R4 : ( 256, 176),
+ GGMLQuantizationType.Q6_K_R4 : ( 256, 210),
+ GGMLQuantizationType.IQ2_XXS_R4 : ( 256, 66),
+ GGMLQuantizationType.IQ2_XS_R4 : ( 256, 74),
+ GGMLQuantizationType.IQ3_XXS_R4 : ( 256, 98),
+ GGMLQuantizationType.IQ1_S_R4 : ( 32, 6),
+ GGMLQuantizationType.IQ4_NL_R4 : ( 32, 18),
+ GGMLQuantizationType.IQ3_S_R4 : ( 256, 110),
+ GGMLQuantizationType.IQ2_S_R4 : ( 256, 82),
+ GGMLQuantizationType.IQ4_XS_R8 : ( 256, 136),
+ GGMLQuantizationType.IQ1_M_R4 : ( 32, 7),
+ GGMLQuantizationType.BF16_R16 : ( 1, 2),
+ GGMLQuantizationType.Q6_0_R4 : ( 32, 26),
+ GGMLQuantizationType.IQ2_BN_R4 : ( 64, 16),
+ GGMLQuantizationType.IQ2_K_R4 : ( 256, 76),
+ GGMLQuantizationType.IQ3_K_R4 : ( 256, 110),
+ GGMLQuantizationType.IQ4_K_R4 : ( 256, 144),
+ GGMLQuantizationType.IQ5_K_R4 : ( 256, 176),
+ GGMLQuantizationType.IQ4_KS_R4 : ( 256, 136),
+ GGMLQuantizationType.Q8_KV_R8 : ( 32, 32),
+ GGMLQuantizationType.Q8_K_R8 : ( 256, 258),
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-18 11:49:47 +0000