第六、七轮:离线 ort/WebGPU、R9 查看器整周期、安全层截止下沉、enc_once 前瞻 bug(feat 动态轴)、UTF-8、断点 RNG 续跑与 --stop-at、R2 池无关性自检、R8 出题缓存、本机自检 selftest、冒烟 smoke;遗留清单并入审查文档第六节

This commit is contained in:
Hark
2026-10-17 20:28:00 +00:00
parent e6388b2f0c
commit fd2f2db991
29 changed files with 715 additions and 127 deletions
Executable
+70
View File
@@ -0,0 +1,70 @@
#!/usr/bin/env bash
# 本机自检(2026-10-18 第七轮,遗留 C-5 / C-6:Windows 生命周期与断点续跑只在沙箱验证过)。约 5–10 分钟。
# 在仓库旁的副本(默认 ../<仓库名>_selftest)里做,不碰本仓库数据。结果写本仓库 ml/runs/selftest.json,每项 pass=true/false:
# gen_resume 生成 6 条一口气 vs 先生成 3 条、把最后一行截掉一半、再续到 6 条 → 两份文件逐字节相同
# prep_rerun 打包两次,第二次「待打包 0 个」
# train_stop 训练 --stop-at 20 → rc 143、last.pt 第 20 步;续训到 40 步 rc 0
# train_stopfile 训练中途写 ml/runs/STOP → 60 s 内 rc 143 且 last.pt 已存(控制台「停止」走的就是这条路)
# train_term Git Bash 的 timeout 送 TERM 给训练 → 是否来得及存 last.pt(Windows 原生 python 未必收得到,结果决定控制台是否只能靠 STOP 文件)
# apg_resume APG --stop-at 5 → rc 143;--resume 续跑 rc 0
# tree_killed / survive_parent_exit 控制台的进程树停止、关窗口后子树存活(ml/selftest_proc.py)
set -o pipefail
cd "$(dirname "$0")/.."
SRC=$(pwd); PY=${PYTHON:-python}; export PYTHONUTF8=1
D=${SELFTEST_DIR:-$(dirname "$SRC")/$(basename "$SRC")_selftest}
rm -rf "$D"; mkdir -p "$D"
git ls-files -z | tar --null -T - -cf - | (cd "$D" && tar xf -)
[ -d ml/node_modules ] && cp -r ml/node_modules "$D/ml/node_modules"
cd "$D"; mkdir -p ml/runs ml/data
R=ml/runs/selftest_items.jsonl; : > $R
rec() { $PY -c "import json,sys;print(json.dumps({'item':sys.argv[1],'pass':json.loads(sys.argv[2]),'detail':sys.argv[3]},ensure_ascii=False))" "$1" "$2" "$3" >> $R; echo "[selftest] $1 → $2 $3"; }
CFGS=ml/configs/smoke.json
# 1 生成续写
node ml/gen/gen_dataset.cjs --n 6 --seed 11 --out ml/data/a.jsonl 2>/dev/null
node ml/gen/gen_dataset.cjs --n 3 --seed 11 --out ml/data/shard_0011.jsonl.part 2>/dev/null
sz=$(wc -c < ml/data/shard_0011.jsonl.part); head -c $((sz - 40)) ml/data/shard_0011.jsonl.part > t && mv t ml/data/shard_0011.jsonl.part
node ml/gen/gen_dataset.cjs --n 6 --seed 11 --out ml/data/shard_0011.jsonl.part 2>ml/runs/st_gen.log; mv ml/data/shard_0011.jsonl.part ml/data/shard_0011.jsonl
cmp -s ml/data/a.jsonl ml/data/shard_0011.jsonl && rec gen_resume true "$(grep -o '续写[^,]*' ml/runs/st_gen.log | head -1)" || rec gen_resume false "续写结果与一口气生成不同"
rm -f ml/data/a.jsonl
node ml/gen/gen_dataset.cjs --n 40 --seed 12 --out ml/data/shard_0012.jsonl 2>/dev/null # 训练自检用的小数据
# 2 打包二次运行
bash ml/prep/run_prep.sh 4 >/dev/null 2>&1; bash ml/prep/run_prep.sh 4 > ml/runs/st_prep.log 2>&1
grep -q "待打包 0 个" ml/runs/st_prep.log && rec prep_rerun true "" || rec prep_rerun false "$(head -c 200 ml/runs/st_prep.log)"
T="--samples ml/data/samples --no-eval --config $CFGS --steps 40"
step_of() { $PY -c "import torch;print(torch.load('$1',map_location='cpu',weights_only=False)['step'])" 2>/dev/null; }
# 3 --stop-at
$PY -m ml.train $T --out ml/runs/st_a --stop-at 20 > ml/runs/st_a.log 2>&1; rc=$?
s=$(step_of ml/runs/st_a/last.pt)
$PY -m ml.train $T --out ml/runs/st_a --resume ml/runs/st_a/last.pt >> ml/runs/st_a.log 2>&1; rc2=$?
[ $rc = 143 ] && [ "$s" = 20 ] && [ $rc2 = 0 ] && rec train_stop true "rc 143 / 第 $s 步 / 续训 rc 0" || rec train_stop false "rc=$rc step=$s 续训rc=$rc2"
# 4 STOP 文件
rm -f ml/runs/STOP
$PY -m ml.train --samples ml/data/samples --no-eval --config $CFGS --steps 1000000 --out ml/runs/st_b > ml/runs/st_b.log 2>&1 & pid=$!
sleep 25; touch ml/runs/STOP; t0=$(date +%s)
wait $pid; rc=$?; dt=$(( $(date +%s) - t0 )); rm -f ml/runs/STOP
[ $rc = 143 ] && [ -f ml/runs/st_b/last.pt ] && rec train_stopfile true "写 STOP 后 ${dt}s 退出,第 $(step_of ml/runs/st_b/last.pt) 步" || rec train_stopfile false "rc=$rc ${dt}s"
# 5 timeout TERM
if command -v timeout >/dev/null; then
timeout -k 30 25 $PY -m ml.train --samples ml/data/samples --no-eval --config $CFGS --steps 1000000 --out ml/runs/st_c > ml/runs/st_c.log 2>&1; rc=$?
grep -q "收到停止请求" ml/runs/st_c.log && rec train_term true "rc=$rc,TERM 送达并存档" || rec train_term false "rc=$rc,TERM 没送到 python(Windows 原生进程常见)→ 阶段超时只能硬杀,进度靠每 10 分钟存档"
else rec train_term false "没有 timeout 命令"; fi
# 6 APG 续跑
$PY -m ml.apg.apg_finetune --init ml/runs/st_a/last.pt --samples ml/data/samples --out ml/runs/st_apg --steps 10 --stop-at 5 > ml/runs/st_apg.log 2>&1; rc=$?
$PY -m ml.apg.apg_finetune --init ml/runs/st_a/last.pt --samples ml/data/samples --out ml/runs/st_apg --steps 10 --resume ml/runs/st_apg/last.pt >> ml/runs/st_apg.log 2>&1; rc2=$?
[ $rc = 143 ] && [ $rc2 = 0 ] && rec apg_resume true "" || rec apg_resume false "rc=$rc 续跑rc=$rc2(日志 $D/ml/runs/st_apg.log)"
# 7 进程树 / 关窗口
pj=$($PY ml/selftest_proc.py 2>ml/runs/st_proc.err | tail -1)
echo "$pj" > ml/runs/st_proc.json
k=$($PY -c "import json,sys;d=json.loads(sys.argv[1]);print(str(d.get('tree_killed')).lower(), str(d.get('survive_parent_exit')).lower())" "$pj" 2>/dev/null || echo "null null")
rec tree_killed ${k% *} "$pj" ; rec survive_parent_exit ${k#* } ""
mkdir -p "$SRC/ml/runs"
$PY - "$R" "$SRC/ml/runs/selftest.json" "$D" <<'PY'
import json, sys
items = []
for l in open(sys.argv[1], encoding='utf-8'):
try: items.append(json.loads(l))
except Exception: items.append({'item': 'parse', 'pass': False, 'detail': l.strip()[:200]})
out = {'all_pass': all(i['pass'] is True for i in items), 'items': items, 'dir': sys.argv[3]}
json.dump(out, open(sys.argv[2], 'w', encoding='utf-8'), indent=1, ensure_ascii=False)
print('[selftest] 全部通过' if out['all_pass'] else '[selftest] 有未通过项,见 ' + sys.argv[2])
PY