41 lines
685 B
JSON
41 lines
685 B
JSON
{
|
|
"model": {
|
|
"d": 192,
|
|
"heads": 8,
|
|
"ff": 768,
|
|
"enc": 4,
|
|
"dec": 4
|
|
},
|
|
"train": {
|
|
"batch": 1024,
|
|
"lr": 0.0001,
|
|
"warmup": 1000,
|
|
"steps": 3000,
|
|
"wd": 0.01,
|
|
"betas": [
|
|
0.9,
|
|
0.95
|
|
],
|
|
"amp": "bf16",
|
|
"grad_clip": 1.0,
|
|
"w_mf": 1.0,
|
|
"w_fk": 0.1,
|
|
"w_smooth": 0.01,
|
|
"w_value": 0.0,
|
|
"w_sub": 0.0,
|
|
"p_neq": 0.5,
|
|
"p_edge": 0.1,
|
|
"p_neq_warm": 0,
|
|
"fk_tmax": 0.3,
|
|
"fk_caps": 96,
|
|
"ema": 0.999,
|
|
"aug_pre_drop": 0.15,
|
|
"aug_pre_sigma": 0.1,
|
|
"aug_dq_sigma": 0.3,
|
|
"ckpt_min": 10,
|
|
"eval_every": 1000,
|
|
"eval_tasks": 256,
|
|
"log_every": 50,
|
|
"patience_evals": 3
|
|
}
|
|
} |