2026-10-17 20:28:00 +00:00
#!/usr/bin/env bash
# 本机自检（2026-10-18 第七轮，遗留 C-5 / C-6：Windows 生命周期与断点续跑只在沙箱验证过）。约 5–10 分钟。
# 在仓库旁的副本（默认 ../<仓库名>_selftest）里做，不碰本仓库数据。结果写本仓库 ml/runs/selftest.json，每项 pass=true/false：
# gen_resume 生成 6 条一口气 vs 先生成 3 条、把最后一行截掉一半、再续到 6 条 → 两份文件逐字节相同
# prep_rerun 打包两次，第二次「待打包 0 个」
# train_stop 训练 --stop-at 20 → rc 143、last.pt 第 20 步；续训到 40 步 rc 0
# train_stopfile 训练中途写 ml/runs/STOP → 60 s 内 rc 143 且 last.pt 已存（控制台「停止」走的就是这条路）
# train_term Git Bash 的 timeout 送 TERM 给训练 → 是否来得及存 last.pt（
# apg_resume APG --stop-at 5 → rc 143；
# tree_killed / survive_parent_exit 控制台的进程树停止、关窗口后子树存活（ml/selftest_proc.py）
set -o pipefail
cd " $( dirname " $0 " ) /.."
SRC = $( pwd ) ; PY = ${ PYTHON :- python } ; export PYTHONUTF8 = 1
D = ${ SELFTEST_DIR :-$( dirname " $SRC " ) / $( basename " $SRC " ) _selftest }
rm -rf " $D " ; mkdir -p " $D "
git ls-files -z | tar --null -T - -cf - | ( cd " $D " && tar xf -)
[ -d ml/node_modules ] && cp -r ml/node_modules " $D /ml/node_modules"
cd " $D " ; mkdir -p ml/runs ml/data
R = ml/runs/selftest_items.jsonl; : > $R
2026-10-18 06:34:21 +08:00
rec() { $PY -c "import json,sys
try: v=json.loads(sys.argv[2])
except Exception: v=False # 值不是合法 JSON 时记 false，不能让整项静默消失（第七轮实测：'none' 抛异常 → 两项没进结果）
print(json.dumps({'item':sys.argv[1],'pass':v,'detail':sys.argv[3]},ensure_ascii=False))" " $1 " " $2 " " $3 " >> $R ; echo "[selftest] $1 → $2 $3 " ; }
2026-10-17 20:28:00 +00:00
CFGS = ml/configs/smoke.json
# 1 生成续写
node ml/gen/gen_dataset.cjs --n 6 --seed 11 --out ml/data/a.jsonl 2>/dev/null
node ml/gen/gen_dataset.cjs --n 3 --seed 11 --out ml/data/shard_0011.jsonl.part 2>/dev/null
sz = $( wc -c < ml/data/shard_0011.jsonl.part) ; head -c $(( sz - 40 )) ml/data/shard_0011.jsonl.part > t && mv t ml/data/shard_0011.jsonl.part
node ml/gen/gen_dataset.cjs --n 6 --seed 11 --out ml/data/shard_0011.jsonl.part 2>ml/runs/st_gen.log; mv ml/data/shard_0011.jsonl.part ml/data/shard_0011.jsonl
2026-10-18 06:34:21 +08:00
gd = $( $PY -c "import json,sys
f=lambda p:[json.dumps({**r,'plan_ms':0,'cert':{**r.get('cert',{}),'ms':0}},sort_keys=True) for r in (json.loads(l) for l in open(p,encoding='utf-8') if l.strip())]
a,b=f(sys.argv[1]),f(sys.argv[2]);print('OK' if a==b else '不同 %d/%d 条'%(sum(x!=y for x,y in zip(a,b))+abs(len(a)-len(b)),max(len(a),len(b))))" ml/data/a.jsonl ml/data/shard_0011.jsonl)
if [ " $gd " = OK ] ; then rec gen_resume true " $( grep -o '续写[^,]*' ml/runs/st_gen.log | head -1) （逐条一致；不含 plan_ms/cert.ms 两个墙钟字段，第七轮实测 cmp -s 恒不等）"
else rec gen_resume false "续写与一口气生成不同： $gd " ; fi
2026-10-17 20:28:00 +00:00
rm -f ml/data/a.jsonl
node ml/gen/gen_dataset.cjs --n 40 --seed 12 --out ml/data/shard_0012.jsonl 2>/dev/null # 训练自检用的小数据
# 2 打包二次运行
bash ml/prep/run_prep.sh 4 >/dev/null 2>& 1; bash ml/prep/run_prep.sh 4 > ml/runs/st_prep.log 2>& 1
grep -q "待打包 0 个" ml/runs/st_prep.log && rec prep_rerun true "" || rec prep_rerun false " $( head -c 200 ml/runs/st_prep.log) "
T = "--samples ml/data/samples --no-eval --config $CFGS --steps 40"
step_of() { $PY -c "import torch;print(torch.load(' $1 ',map_location='cpu',weights_only=False)['step'])" 2>/dev/null; }
# 3 --stop-at
$PY -m ml.train $T --out ml/runs/st_a --stop-at 20 > ml/runs/st_a.log 2>& 1; rc = $?
s = $( step_of ml/runs/st_a/last.pt)
$PY -m ml.train $T --out ml/runs/st_a --resume ml/runs/st_a/last.pt >> ml/runs/st_a.log 2>& 1; rc2 = $?
[ $rc = 143 ] && [ " $s " = 20 ] && [ $rc2 = 0 ] && rec train_stop true "rc 143 / 第 $s 步 / 续训 rc 0" || rec train_stop false "rc= $rc step= $s 续训rc= $rc2 "
# 4 STOP 文件
rm -f ml/runs/STOP
$PY -m ml.train --samples ml/data/samples --no-eval --config $CFGS --steps 1000000 --out ml/runs/st_b > ml/runs/st_b.log 2>& 1 & pid = $!
sleep 25; touch ml/runs/STOP; t0 = $( date +%s)
wait $pid ; rc = $? ; dt = $(( $( date +%s) - t0 )) ; rm -f ml/runs/STOP
[ $rc = 143 ] && [ -f ml/runs/st_b/last.pt ] && rec train_stopfile true "写 STOP 后 ${ dt } s 退出，第 $( step_of ml/runs/st_b/last.pt) 步" || rec train_stopfile false "rc= $rc ${ dt } s"
# 5 timeout TERM
if command -v timeout >/dev/null; then
timeout -k 30 25 $PY -m ml.train --samples ml/data/samples --no-eval --config $CFGS --steps 1000000 --out ml/runs/st_c > ml/runs/st_c.log 2>& 1; rc = $?
grep -q "收到停止请求" ml/runs/st_c.log && rec train_term true "rc= $rc ， || rec train_term false "rc= $rc ， （
else rec train_term false "没有 timeout 命令" ; fi
# 6 APG 续跑
2026-10-18 06:34:21 +08:00
$PY -m ml.apg.apg_finetune --config $CFGS --init ml/runs/st_a/last.pt --samples ml/data/samples --out ml/runs/st_apg --steps 10 --stop-at 5 > ml/runs/st_apg.log 2>& 1; rc = $?
$PY -m ml.apg.apg_finetune --config $CFGS --init ml/runs/st_a/last.pt --samples ml/data/samples --out ml/runs/st_apg --steps 10 --resume ml/runs/st_apg/last.pt >> ml/runs/st_apg.log 2>& 1; rc2 = $?
2026-10-17 20:28:00 +00:00
[ $rc = 143 ] && [ $rc2 = 0 ] && rec apg_resume true "" || rec apg_resume false "rc= $rc 续跑rc= $rc2 （日志 $D /ml/runs/st_apg.log）
# 7 进程树 / 关窗口
pj = $( $PY ml/selftest_proc.py 2>ml/runs/st_proc.err | tail -1)
echo " $pj " > ml/runs/st_proc.json
2026-10-18 06:34:21 +08:00
k = $( $PY -c "import json,sys
try: d=json.loads(sys.argv[1])
except Exception: d={}
print(json.dumps(d.get('tree_killed')), json.dumps(d.get('survive_parent_exit')))" " $pj " 2>/dev/null || echo "null null" )
2026-10-17 20:28:00 +00:00
rec tree_killed ${ k % * } " $pj " ; rec survive_parent_exit ${ k #* } ""
mkdir -p " $SRC /ml/runs"
$PY - " $R " " $SRC /ml/runs/selftest.json" " $D " <<'PY'
import json, sys
items = []
for l in open(sys.argv[1], encoding='utf-8'):
try: items.append(json.loads(l))
except Exception: items.append({'item': 'parse', 'pass': False, 'detail': l.strip()[:200]})
out = {'all_pass': all(i['pass'] is True for i in items), 'items': items, 'dir': sys.argv[3]}
json.dump(out, open(sys.argv[2], 'w', encoding='utf-8'), indent=1, ensure_ascii=False)
print('[selftest] 全部通过' if out['all_pass'] else '[selftest] 有未通过项，见 ' + sys.argv[2])
PY
