11 lines
436 B
JSON
11 lines
436 B
JSON
{
|
|
"model": {"d": 192, "heads": 8, "ff": 768, "enc": 4, "dec": 4, "head": "wta", "K": 8},
|
|
"train": {
|
|
"batch": 1024, "lr": 1e-3, "warmup": 1000, "steps": 60000, "wd": 0.01, "betas": [0.9, 0.95],
|
|
"amp": "bf16", "grad_clip": 1.0,
|
|
"w_mf": 1.0, "w_fk": 0.5, "w_smooth": 0.05, "w_value": 0.1, "w_sub": 0.1,
|
|
"p_neq": 0.25, "fk_caps": 96,
|
|
"eval_every": 2000, "eval_tasks": 512, "log_every": 50, "patience_evals": 5
|
|
}
|
|
}
|