ml: 第二轮审查 B-1~B-8 处置 + 一键训练控制台
- B-1 运行时投影护栏恢复 5 mm；老师轨迹按认证最小间隙 < 3 mm 剔除（stats/build_dataset，gapDrop） - B-2 APG 端点+中点 hinge 取均值；B-3 check_r2 空匹配保护、5000×50000、分块 256，pass=false 时 prep 停下 - B-4 R7 按全部 B/C 题统计；B-5 r8 起点贴柱；B-7 pipeline r4（export_onnx --random，训练前测延迟） - B-8 起步失败 ≤0.2%、column_gap 分块、pipefail、门禁结果先写 .tmp - 训练控制台：ml/console.bat|sh → ml/monitor.py（开始/停止/进度/日志接口）+ monitor.html； pipeline.sh all 支持 FROM/TO，阶段状态写 ml/runs/pipeline.json（ml/pstate.py）；训练/APG 日志加 total、t - 文档：README 一键运行与训练控制台；审查文档追加第三轮处置；方案余量决策与 R4 时点
This commit is contained in:
Hark
+40
-7
@@ -2,6 +2,7 @@
|
||||
# 一条龙：bash ml/pipeline.sh <阶段> ；阶段按顺序：
|
||||
# setup 安装 Python/Node 依赖（torch 需先按 CUDA 版本装好）
|
||||
# kin 导出运动学/碰撞模型 + JS↔PyTorch 对拍（必须 PASS）
|
||||
# r4 浏览器推理延迟摸底（随机初始化模型导出 ONNX；训练前做，不过就改模型规模，不必等训完）
|
||||
# gate1 老师一致性门禁（决定生成时是否加 --consensus 4）
|
||||
# gen 生成老师轨迹（NPROC 进程；CONSENSUS=1 时加共识过滤；样本不足 TARGET_SAMPLES 自动补生成）
|
||||
# prep 打包训练样本 + R2 标签一致性检查
|
||||
@@ -12,12 +13,13 @@
|
||||
# export 导出 ONNX（fp32）到 webapp/models/
|
||||
# bench 部署形态评测（Node，策略 + 安全层，含 R8 对抗集）
|
||||
# accept 验收汇总（方案 §9.2 各项 + R2/R7/R8/R10）→ ml/runs/acceptance.json
|
||||
# all 依次执行 kin → accept
|
||||
# all 一键依次执行 kin → accept（FROM=gen TO=train 可只跑一段），并在后台自动打开训练监控（r4 需在浏览器里看，不含在 all 里；R2、M5 不通过会停下）
|
||||
# monitor 训练控制台网页 http://localhost:8100：点「开始训练」一键跑全流程、看进度、可停止（也可双击 ml/console.bat / ml/console.sh）
|
||||
# 可调环境变量：PYTHON（默认 python；Linux 无 python 时设 PYTHON=python3）、NPROC、N_TASKS、TARGET_SAMPLES、
|
||||
# CFG（ml/configs/base.json | wta.json）、CKPT（覆盖后续阶段的起点模型）、CONSENSUS
|
||||
# 2026-10-18：按另一台机器的审查改——头分目录不互相覆盖、统一 $PYTHON（Windows Git Bash 可跑）、
|
||||
# gen 自动补量、m5 双口径、export 默认 fp32、新增 accept。
|
||||
set -e
|
||||
set -e -o pipefail # pipefail：`node ... | tee` 里 node 失败也算失败（否则门禁中断会留下残缺的 consistency.json 继续往下跑）
|
||||
cd "$(dirname "$0")/.."
|
||||
PY=${PYTHON:-python}
|
||||
command -v "$PY" >/dev/null 2>&1 && "$PY" -c "import sys" 2>/dev/null || { echo "找不到可用的 Python（$PY），请设 PYTHON=..."; exit 1; }
|
||||
@@ -31,10 +33,21 @@ SEL=ml/runs/selected_ckpt
|
||||
ckpt() { if [ -n "${CKPT:-}" ]; then echo "$CKPT"; elif [ -f $SEL ]; then cat $SEL; else echo ml/runs/$TAG/best.pt; fi; }
|
||||
stage=${1:-help}
|
||||
run() { echo "== $*"; "$@"; }
|
||||
# 2026-10-18：每个阶段开始/结束写 ml/runs/pipeline.json（监控网页据此显示全流程进度与各步用时）
|
||||
stage_info() { case $1 in
|
||||
gen) echo "target_tasks=$(( (N_TASKS + NPROC - 1) / NPROC * NPROC ))" ;;
|
||||
train) echo "tag=$TAG bench_steps=2000" ;;
|
||||
m5) echo "tag=$TAG" ;;
|
||||
mine) echo "rounds=${ROUNDS:-6} apg_steps=${APG_STEPS:-1000}" ;;
|
||||
esac; }
|
||||
case " setup kin gate1 gen prep train m5 apg mine export bench accept r4 " in *" $stage "*)
|
||||
$PY -m ml.pstate begin $stage $(stage_info $stage) || true
|
||||
trap 'rc=$?; $PY -m ml.pstate end '"$stage"' $rc >/dev/null 2>&1 || true' EXIT ;;
|
||||
esac
|
||||
case $stage in
|
||||
setup) run $PY -m pip install -r ml/requirements.txt; (cd ml && npm install --no-audit --no-fund) ;;
|
||||
kin) run node ml/prep/export_kin.cjs; run node ml/eval/dump_parity.cjs 2000; run $PY -m ml.eval.check_parity ;;
|
||||
gate1) mkdir -p ml/data; run node ml/gen/consistency.cjs 300 4242 | tee ml/data/consistency.json ;;
|
||||
gate1) mkdir -p ml/data; run node ml/gen/consistency.cjs 300 4242 | tee ml/data/consistency.json.tmp; mv ml/data/consistency.json.tmp ml/data/consistency.json ;; # 中断时不留残缺结果
|
||||
gen) per=$(( (N_TASKS + NPROC - 1) / NPROC ))
|
||||
if [ -z "${CONSENSUS:-}" ] && [ -f ml/data/consistency.json ]; then
|
||||
CONSENSUS=$($PY -c "import json;print(1 if json.load(open('ml/data/consistency.json'))['consistentByGateRule']<0.9 else 0)")
|
||||
@@ -48,11 +61,15 @@ case $stage in
|
||||
need=$($PY -c "import json,math;s=json.load(open('ml/data/stats.json'));d=$TARGET_SAMPLES-s['samplesEvery3'];print(0 if d<=0 else math.ceil(d/(s['samplesEvery3']/s['tasks'])*1.05))")
|
||||
[ "$need" = "0" ] && break
|
||||
echo "样本不足目标 $TARGET_SAMPLES，补生成 $need 条任务"
|
||||
run bash ml/gen/run_parallel.sh $(( (need + NPROC - 1) / NPROC )) $NPROC $s0 "$extra"
|
||||
per2=$(( (need + NPROC - 1) / NPROC ))
|
||||
$PY -m ml.pstate set gen target_tasks=$(( $($PY -c "import json;print(json.load(open('ml/runs/pipeline.json'))['stages']['gen']['info']['target_tasks'])") + per2 * NPROC )) || true
|
||||
run bash ml/gen/run_parallel.sh $per2 $NPROC $s0 "$extra"
|
||||
s0=$((s0 + NPROC))
|
||||
done ;;
|
||||
prep) run bash ml/prep/run_prep.sh $NPROC
|
||||
run $PY -m ml.eval.check_r2 --samples ml/data/samples --out ml/data/r2.json ;;
|
||||
run $PY -m ml.eval.check_r2 --samples ml/data/samples --out ml/data/r2.json
|
||||
# R2 明确不通过（pass=false）就停在这里，不让一键流程白训 3 小时；近邻对不足（pass=null）不拦
|
||||
$PY -c "import json,sys;sys.exit(4 if json.load(open('ml/data/r2.json')).get('pass') is False else 0)" || { echo "R2 不通过：近邻样本标签不自洽，先把 ml/data/r2.json 发我；确认要继续就从「预训练」开始（FROM=train）"; exit 4; } ;;
|
||||
train) run $PY -m ml.train --samples ml/data/samples --out ml/runs/bench_$TAG --steps 2000 --no-eval --config $CFG
|
||||
run $PY -m ml.train --samples ml/data/samples --out ml/runs/$TAG --config $CFG ;;
|
||||
m5) [ -f ml/runs/nc_gate.jsonl ] || run $PY -m ml.mining.mine_round --make-gate 1000 --seed 4321 --mix 0,0,1,0 --name nc_gate.jsonl --out ml/runs
|
||||
@@ -80,11 +97,27 @@ PY
|
||||
;;
|
||||
apg) ck=$(ckpt); run $PY -m ml.apg.apg_finetune --init $ck --samples ml/data/samples --out ml/runs/apg; cp ml/runs/apg/best.pt ml/runs/current.pt ;;
|
||||
mine) run bash ml/mining/loop.sh ${ROUNDS:-6} ${APG_STEPS:-1000} ${MINE_N:-50000} ;;
|
||||
r4) run $PY -m ml.export_onnx --random $CFG --out webapp/models/traj_policy.onnx
|
||||
echo "下一步：python ml/serve.py → 浏览器打开 http://localhost:8000/?policy=1 → 拖动黄球 1 分钟 → 控制台 __POLICY.latency()" ;;
|
||||
monitor) run $PY -m ml.monitor ${MONITOR_ARGS:-} ;;
|
||||
export) ck=ml/runs/current.pt; [ -f $ck ] || ck=$(ckpt)
|
||||
run $PY -m ml.export_onnx --ckpt $ck --out webapp/models/traj_policy.onnx ;;
|
||||
bench) run node tests/policy_bench.cjs webapp/models/traj_policy.onnx nc,route,r8 --nc1000 --out ml/runs/policy_bench.json ;;
|
||||
accept) ck=ml/runs/current.pt; [ -f $ck ] || ck=$(ckpt)
|
||||
run $PY -m ml.eval.acceptance --ckpt $ck --samples ml/data/samples --out ml/runs/acceptance.json ;;
|
||||
all) for s in kin gate1 gen prep train m5 apg mine export bench accept; do bash ml/pipeline.sh $s; done ;;
|
||||
*) sed -n '2,20p' "$0" ;;
|
||||
all) # FROM / TO 指定起止阶段（默认 kin → accept），网页控制台的「开始训练」也走这里
|
||||
FROM=${FROM:-kin}; TO=${TO:-accept}; PLAN=""; on=0
|
||||
for s in setup kin gate1 gen prep train m5 apg mine export bench accept; do
|
||||
[ "$s" = "$FROM" ] && on=1; [ $on = 1 ] && PLAN="$PLAN $s"; [ "$s" = "$TO" ] && break
|
||||
done
|
||||
PLAN=${PLAN# }; [ -n "$PLAN" ] || { echo "FROM=$FROM / TO=$TO 不是有效阶段"; exit 1; }
|
||||
echo "计划：$PLAN"
|
||||
$PY -m ml.pstate reset ${PLAN// /,}
|
||||
# 监控网页随 all 自动在后台启动（MONITOR=0 关闭）；端口已被占用说明已经开着，不重复启动
|
||||
if [ "${MONITOR:-1}" = "1" ] && $PY -c "import socket,sys;s=socket.socket();sys.exit(0 if s.connect_ex(('127.0.0.1',8100)) else 1)"; then
|
||||
mkdir -p ml/runs; $PY -m ml.monitor ${MONITOR_ARGS:-} > ml/runs/monitor.log 2>&1 &
|
||||
echo "训练监控已在后台启动：http://localhost:8100/ （全部跑完后仍可查看，关闭终端即结束）"
|
||||
fi
|
||||
for s in $PLAN; do bash ml/pipeline.sh $s; done ;;
|
||||
*) sed -n '2,22p' "$0" ;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user