forked from jiuyuan/CPM-9G-8B
15 lines
266 B
JSON
15 lines
266 B
JSON
|
{
|
||
|
|
"vocab_size": 119696,
|
||
|
|
"dropout_p": 0.0,
|
||
|
|
"eps": 1e-05,
|
||
|
|
"half": true,
|
||
|
|
"dim_model": 4096,
|
||
|
|
"dim_ff": 14336,
|
||
|
|
"num_layers": 32,
|
||
|
|
"num_heads": 32,
|
||
|
|
"num_kv_heads": 32,
|
||
|
|
"dim_head": 128,
|
||
|
|
"activate_fn": "silu",
|
||
|
|
"scale": false
|
||
|
|
}
|