quant_config.json

{
  "format": "auto-quant-v1",
  "bits": 8,
  "group_size": null,
  "symmetric": true,
  "q_range": [
    -127,
    127
  ],
  "scale_dtype": "float16",
  "granularity": "per output channel / embedding row",
  "modules": [
    "model.embeddings.tok_embeddings",
    "model.layers.0.attn.Wqkv",
    "model.layers.0.attn.Wo",
    "model.layers.0.mlp.Wi",
    "model.layers.0.mlp.Wo",
    "model.layers.1.attn.Wqkv",
    "model.layers.1.attn.Wo",
    "model.layers.1.mlp.Wi",
    "model.layers.1.mlp.Wo",
    "model.layers.2.attn.Wqkv",
    "model.layers.2.attn.Wo",
    "model.layers.2.mlp.Wi",
    "model.layers.2.mlp.Wo",
    "model.layers.3.attn.Wqkv",
    "model.layers.3.attn.Wo",
    "model.layers.3.mlp.Wi",
    "model.layers.3.mlp.Wo",
    "model.layers.4.attn.Wqkv",
    "model.layers.4.attn.Wo",
    "model.layers.4.mlp.Wi",
    "model.layers.4.mlp.Wo",
    "model.layers.5.attn.Wqkv",
    "model.layers.5.attn.Wo",
    "model.layers.5.mlp.Wi",
    "model.layers.5.mlp.Wo",
    "model.layers.6.attn.Wqkv",
    "model.layers.6.attn.Wo",
    "model.layers.6.mlp.Wi",
    "model.layers.6.mlp.Wo",
    "model.layers.7.attn.Wqkv",
    "model.layers.7.attn.Wo",
    "model.layers.7.mlp.Wi",
    "model.layers.7.mlp.Wo",
    "model.layers.8.attn.Wqkv",
    "model.layers.8.attn.Wo",
    "model.layers.8.mlp.Wi",
    "model.layers.8.mlp.Wo",
    "model.layers.9.attn.Wqkv",
    "model.layers.9.attn.Wo",
    "model.layers.9.mlp.Wi",
    "model.layers.9.mlp.Wo",
    "model.layers.10.attn.Wqkv",
    "model.layers.10.attn.Wo",
    "model.layers.10.mlp.Wi",
    "model.layers.10.mlp.Wo",
    "model.layers.11.attn.Wqkv",
    "model.layers.11.attn.Wo",
    "model.layers.11.mlp.Wi",
    "model.layers.11.mlp.Wo",
    "model.layers.12.attn.Wqkv",
    "model.layers.12.attn.Wo",
    "model.layers.12.mlp.Wi",
    "model.layers.12.mlp.Wo",
    "model.layers.13.attn.Wqkv",
    "model.layers.13.attn.Wo",
    "model.layers.13.mlp.Wi",
    "model.layers.13.mlp.Wo",
    "model.layers.14.attn.Wqkv",
    "model.layers.14.attn.Wo",
    "model.layers.14.mlp.Wi",
    "model.layers.14.mlp.Wo",
    "model.layers.15.attn.Wqkv",
    "model.layers.15.attn.Wo",
    "model.layers.15.mlp.Wi",
    "model.layers.15.mlp.Wo",
    "model.layers.16.attn.Wqkv",
    "model.layers.16.attn.Wo",
    "model.layers.16.mlp.Wi",
    "model.layers.16.mlp.Wo",
    "model.layers.17.attn.Wqkv",
    "model.layers.17.attn.Wo",
    "model.layers.17.mlp.Wi",
    "model.layers.17.mlp.Wo",
    "model.layers.18.attn.Wqkv",
    "model.layers.18.attn.Wo",
    "model.layers.18.mlp.Wi",
    "model.layers.18.mlp.Wo",
    "model.layers.19.attn.Wqkv",
    "model.layers.19.attn.Wo",
    "model.layers.19.mlp.Wi",
    "model.layers.19.mlp.Wo",
    "model.layers.20.attn.Wqkv",
    "model.layers.20.attn.Wo",
    "model.layers.20.mlp.Wi",
    "model.layers.20.mlp.Wo",
    "model.layers.21.attn.Wqkv",
    "model.layers.21.attn.Wo",
    "model.layers.21.mlp.Wi",
    "model.layers.21.mlp.Wo",
    "head.dense"
  ],
  "weights": "auto_quant_int8.safetensors",
  "activations": "unquantized (device float dtype)"
}