{
"vocab_size": 119696,
"dropout_p": 0.0,
"eps": 1e-05,
"half": true,
"use_flash_attn": false,
"flash_attn_mask_shape": "1d",
"dim_model": 4096,
"dim_ff": 12288,
"dim_head": 128,
"num_heads": 32,
"num_kv_heads": 32,
"num_layers": 32,
"activate_fn": "silu",
"scale": false
}