{
"vocab_size": 119696,
"dropout_p": 0.0,
"eps": 1e-05,
"half": true,
"dim_model": 4096,
"dim_ff": 14336,
"num_layers": 32,
"num_heads": 32,
"num_kv_heads": 32,
"dim_head": 128,
"activate_fn": "silu",
"scale": false
}