ml: 按运行手册审查修订——头分目录不互相覆盖（ml/runs/<CFG>、selected_ckpt/CKPT）；统一 $PYTHON（Windows Git Bash 可跑）；gen 按实测样本/任务自动补量；R10 改终态判据（≥5k 样本/s≈60k 步≤3.5h）；m5 纯策略/仿真安全层双口径；无老师题定长 6 s 口径；APG 展开随批时长、端点+中点铰链、余量 3 mm（投影同步）；挖错池只用本轮；门禁加 R7；bench 加 R8 对抗集与刹停率、--nc-n/--route-n/--r8-n；新增 check_r2 与 acceptance；export 默认 fp32；出题分块防 OOM；val 为空保护
This commit is contained in:
Hark
+66
-27
@@ -3,49 +3,88 @@
|
||||
# setup 安装 Python/Node 依赖（torch 需先按 CUDA 版本装好）
|
||||
# kin 导出运动学/碰撞模型 + JS↔PyTorch 对拍（必须 PASS）
|
||||
# gate1 老师一致性门禁（决定生成时是否加 --consensus 4）
|
||||
# gen 生成 5 万条老师轨迹（NPROC 进程；CONSENSUS=1 时加共识过滤）
|
||||
# prep 打包训练样本
|
||||
# train 预训练（先 2000 步摸吞吐，再正式训练）
|
||||
# m5 MeanFlow 头门禁：贴立柱集关安全层穿柱率 ≤ 2%（不过 → 用 configs/wta.json 重训）
|
||||
# apg APG 首轮微调
|
||||
# gen 生成老师轨迹（NPROC 进程；CONSENSUS=1 时加共识过滤；样本不足 TARGET_SAMPLES 自动补生成）
|
||||
# prep 打包训练样本 + R2 标签一致性检查
|
||||
# train 预训练（先 2000 步摸吞吐，再正式训练）；输出目录随头分开：ml/runs/<CFG 名>
|
||||
# m5 头门禁：贴立柱集穿柱率 ≤ 2%（纯策略口径），同时报仿真安全层口径；通过则记为选定模型
|
||||
# apg APG 首轮微调（从选定模型开始）
|
||||
# mine 挖错循环（3–6 轮，含门禁回滚）
|
||||
# export 导出 ONNX（fp16）到 webapp/models/
|
||||
# bench 部署形态评测（Node，策略 + 安全层）
|
||||
# all 依次执行 kin → bench
|
||||
# export 导出 ONNX（fp32）到 webapp/models/
|
||||
# bench 部署形态评测（Node，策略 + 安全层，含 R8 对抗集）
|
||||
# accept 验收汇总（方案 §9.2 各项 + R2/R7/R8/R10）→ ml/runs/acceptance.json
|
||||
# all 依次执行 kin → accept
|
||||
# 可调环境变量：PYTHON（默认 python；Linux 无 python 时设 PYTHON=python3）、NPROC、N_TASKS、TARGET_SAMPLES、
|
||||
# CFG（ml/configs/base.json | wta.json）、CKPT（覆盖后续阶段的起点模型）、CONSENSUS
|
||||
# 2026-10-18：按另一台机器的审查改——头分目录不互相覆盖、统一 $PYTHON（Windows Git Bash 可跑）、
|
||||
# gen 自动补量、m5 双口径、export 默认 fp32、新增 accept。
|
||||
set -e
|
||||
cd "$(dirname "$0")/.."
|
||||
PY=${PYTHON:-python}
|
||||
command -v "$PY" >/dev/null 2>&1 && "$PY" -c "import sys" 2>/dev/null || { echo "找不到可用的 Python（$PY），请设 PYTHON=..."; exit 1; }
|
||||
export PYTHON=$PY
|
||||
NPROC=${NPROC:-$(nproc)}
|
||||
N_TASKS=${N_TASKS:-50000}
|
||||
TARGET_SAMPLES=${TARGET_SAMPLES:-1700000}
|
||||
CFG=${CFG:-ml/configs/base.json}
|
||||
TAG=$(basename "$CFG" .json)
|
||||
SEL=ml/runs/selected_ckpt
|
||||
ckpt() { if [ -n "${CKPT:-}" ]; then echo "$CKPT"; elif [ -f $SEL ]; then cat $SEL; else echo ml/runs/$TAG/best.pt; fi; }
|
||||
stage=${1:-help}
|
||||
run() { echo "== $*"; "$@"; }
|
||||
case $stage in
|
||||
setup) run pip install -r ml/requirements.txt; (cd ml && npm install --no-audit --no-fund) ;;
|
||||
kin) run node ml/prep/export_kin.cjs; run node ml/eval/dump_parity.cjs 2000; run python -m ml.eval.check_parity ;;
|
||||
setup) run $PY -m pip install -r ml/requirements.txt; (cd ml && npm install --no-audit --no-fund) ;;
|
||||
kin) run node ml/prep/export_kin.cjs; run node ml/eval/dump_parity.cjs 2000; run $PY -m ml.eval.check_parity ;;
|
||||
gate1) mkdir -p ml/data; run node ml/gen/consistency.cjs 300 4242 | tee ml/data/consistency.json ;;
|
||||
gen) per=$(( (N_TASKS + NPROC - 1) / NPROC ))
|
||||
if [ -z "${CONSENSUS:-}" ] && [ -f ml/data/consistency.json ]; then
|
||||
CONSENSUS=$(python -c "import json;print(1 if json.load(open('ml/data/consistency.json'))['consistentByGateRule']<0.9 else 0)")
|
||||
CONSENSUS=$($PY -c "import json;print(1 if json.load(open('ml/data/consistency.json'))['consistentByGateRule']<0.9 else 0)")
|
||||
echo "一致性门禁结果 → CONSENSUS=$CONSENSUS"
|
||||
fi
|
||||
extra=""; [ "${CONSENSUS:-0}" = "1" ] && extra="--consensus 4"
|
||||
run bash ml/gen/run_parallel.sh $per $NPROC 1 "$extra" ;;
|
||||
prep) run bash ml/prep/run_prep.sh $NPROC ;;
|
||||
train) run python -m ml.train --samples ml/data/samples --out ml/runs/bench --steps 2000 --no-eval --config $CFG
|
||||
run python -m ml.train --samples ml/data/samples --out ml/runs/base --config $CFG ;;
|
||||
m5) [ -f ml/runs/nc_gate.jsonl ] || run python -m ml.mining.mine_round --make-gate 1000 --seed 4321 --mix 0,0,1,0 --name nc_gate.jsonl --out ml/runs
|
||||
run python -m ml.sim.closed_loop --ckpt ml/runs/base/best.pt --tasks ml/runs/nc_gate.jsonl --n 1000 --out ml/runs/m5_gate.json
|
||||
python - <<'PY'
|
||||
import json; r = json.load(open('ml/runs/m5_gate.json'))
|
||||
print('贴立柱集（关安全层）穿柱率 %.2f%%，到达率 %.2f%%' % (100 * r['collision_rate'], 100 * r['success']))
|
||||
print('M5 门禁：' + ('通过' if r['collision_rate'] <= 0.02 else '不通过 → CFG=ml/configs/wta.json bash ml/pipeline.sh train'))
|
||||
run bash ml/gen/run_parallel.sh $per $NPROC 1 "$extra"
|
||||
# 补量：共识过滤会降低保留率，样本不足 TARGET_SAMPLES 时按实测"样本/任务"补生成（最多 2 次）
|
||||
s0=$((1 + NPROC))
|
||||
for k in 1 2; do
|
||||
need=$($PY -c "import json,math;s=json.load(open('ml/data/stats.json'));d=$TARGET_SAMPLES-s['samplesEvery3'];print(0 if d<=0 else math.ceil(d/(s['samplesEvery3']/s['tasks'])*1.05))")
|
||||
[ "$need" = "0" ] && break
|
||||
echo "样本不足目标 $TARGET_SAMPLES，补生成 $need 条任务"
|
||||
run bash ml/gen/run_parallel.sh $(( (need + NPROC - 1) / NPROC )) $NPROC $s0 "$extra"
|
||||
s0=$((s0 + NPROC))
|
||||
done ;;
|
||||
prep) run bash ml/prep/run_prep.sh $NPROC
|
||||
run $PY -m ml.eval.check_r2 --samples ml/data/samples --out ml/data/r2.json ;;
|
||||
train) run $PY -m ml.train --samples ml/data/samples --out ml/runs/bench_$TAG --steps 2000 --no-eval --config $CFG
|
||||
run $PY -m ml.train --samples ml/data/samples --out ml/runs/$TAG --config $CFG ;;
|
||||
m5) [ -f ml/runs/nc_gate.jsonl ] || run $PY -m ml.mining.mine_round --make-gate 1000 --seed 4321 --mix 0,0,1,0 --name nc_gate.jsonl --out ml/runs
|
||||
ck=${CKPT:-ml/runs/$TAG/best.pt}
|
||||
run $PY -m ml.sim.closed_loop --ckpt $ck --tasks ml/runs/nc_gate.jsonl --n 1000 --out ml/runs/m5_${TAG}_policy.json
|
||||
run $PY -m ml.sim.closed_loop --ckpt $ck --tasks ml/runs/nc_gate.jsonl --n 1000 --safety --out ml/runs/m5_${TAG}_safety.json
|
||||
$PY - "$TAG" "$ck" <<'PY'
|
||||
import json, sys
|
||||
tag, ck = sys.argv[1], sys.argv[2]
|
||||
p = json.load(open(f'ml/runs/m5_{tag}_policy.json')); s = json.load(open(f'ml/runs/m5_{tag}_safety.json'))
|
||||
ok = p['collision_rate'] <= 0.02
|
||||
out = {'head': tag, 'ckpt': ck, 'pass': ok,
|
||||
'policy_only': {k: p[k] for k in ('n', 'success', 'collision_rate', 'start_fail')},
|
||||
'with_sim_safety': {k: s[k] for k in ('n', 'success', 'collision_rate', 'brakes_mean', 'start_fail')},
|
||||
'note': '门禁只看 policy_only（纯策略穿柱率）；with_sim_safety 为仿真近似安全层，部署口径以 bench 为准'}
|
||||
json.dump(out, open(f'ml/runs/m5_gate_{tag}.json', 'w'), indent=1)
|
||||
print('[%s] 纯策略穿柱率 %.2f%%、到达 %.2f%%；加仿真安全层 穿柱 %.2f%%、到达 %.2f%%' % (
|
||||
tag, 100 * p['collision_rate'], 100 * p['success'], 100 * s['collision_rate'], 100 * s['success']))
|
||||
if ok:
|
||||
open('ml/runs/selected_ckpt', 'w').write(ck); print('M5 门禁：通过 → 后续阶段使用', ck)
|
||||
else:
|
||||
print('M5 门禁：不通过' + (' → CFG=ml/configs/wta.json bash ml/pipeline.sh train && CFG=ml/configs/wta.json bash ml/pipeline.sh m5' if tag != 'wta' else '（两种头都未过，把 m5_gate_*.json 发我）'))
|
||||
sys.exit(3)
|
||||
PY
|
||||
;;
|
||||
apg) run python -m ml.apg.apg_finetune --init ml/runs/base/best.pt --samples ml/data/samples --out ml/runs/apg; cp ml/runs/apg/best.pt ml/runs/current.pt ;;
|
||||
apg) ck=$(ckpt); run $PY -m ml.apg.apg_finetune --init $ck --samples ml/data/samples --out ml/runs/apg; cp ml/runs/apg/best.pt ml/runs/current.pt ;;
|
||||
mine) run bash ml/mining/loop.sh ${ROUNDS:-6} ${APG_STEPS:-1000} ${MINE_N:-50000} ;;
|
||||
export) ck=ml/runs/current.pt; [ -f $ck ] || ck=ml/runs/base/best.pt
|
||||
run python -m ml.export_onnx --ckpt $ck --out webapp/models/traj_policy.onnx --fp16 ;;
|
||||
bench) run node tests/policy_bench.cjs webapp/models/traj_policy.onnx nc,route --nc1000 --out ml/runs/policy_bench.json ;;
|
||||
all) for s in kin gate1 gen prep train m5 apg mine export bench; do bash ml/pipeline.sh $s; done ;;
|
||||
*) sed -n '2,16p' "$0" ;;
|
||||
export) ck=ml/runs/current.pt; [ -f $ck ] || ck=$(ckpt)
|
||||
run $PY -m ml.export_onnx --ckpt $ck --out webapp/models/traj_policy.onnx ;;
|
||||
bench) run node tests/policy_bench.cjs webapp/models/traj_policy.onnx nc,route,r8 --nc1000 --out ml/runs/policy_bench.json ;;
|
||||
accept) ck=ml/runs/current.pt; [ -f $ck ] || ck=$(ckpt)
|
||||
run $PY -m ml.eval.acceptance --ckpt $ck --samples ml/data/samples --out ml/runs/acceptance.json ;;
|
||||
all) for s in kin gate1 gen prep train m5 apg mine export bench accept; do bash ml/pipeline.sh $s; done ;;
|
||||
*) sed -n '2,20p' "$0" ;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user