ml: 按运行手册审查修订——头分目录不互相覆盖(ml/runs/<CFG>、selected_ckpt/CKPT);统一 $PYTHON(Windows Git Bash 可跑);gen 按实测样本/任务自动补量;R10 改终态判据(≥5k 样本/s≈60k 步≤3.5h);m5 纯策略/仿真安全层双口径;无老师题定长 6 s 口径;APG 展开随批时长、端点+中点铰链、余量 3 mm(投影同步);挖错池只用本轮;门禁加 R7;bench 加 R8 对抗集与刹停率、--nc-n/--route-n/--r8-n;新增 check_r2 与 acceptance;export 默认 fp32;出题分块防 OOM;val 为空保护

This commit is contained in:
Hark
2026-10-18 06:40:24 +08:00
committed by cbb
parent 8800a68cad
commit f8cdabb781
18 changed files with 493 additions and 156 deletions
+104
View File
@@ -0,0 +1,104 @@
"""验收汇总(方案 §9.2 + §10.1 R2/R7/R8/R10):把各阶段产物里的判据集中核对,输出 ml/runs/acceptance.json。
python -m ml.eval.acceptance --ckpt ml/runs/current.pt --samples ml/data/samples --out ml/runs/acceptance.json
自己跑的一项:held-out(val 划分)全部任务闭环仿真(开仿真安全层)——可达任务到达率 ≥ 98%、D 类停靠距离 ≤ 老师 + 5 mm、起步失败 = 0。
其余项读已有文件(缺文件记为 missing,不算通过):
ml/runs/policy_bench.json 部署形态:各集合穿模帧 = 0;near 30/30(pos、pose);route 全部到达;nc1000 0 失败;刹停率 < 2%;R8 恢复 ≥ 95%、活锁 = 0
ml/data/r2.json R2 近邻标签 std < 10% 步长
ml/runs/bench_*/log.jsonl R10 吞吐 ≥ 5000 样本/s(≈ 60000 步 ≤ 3.5 h)
ml/runs/m5_gate_*.json M5 头门禁
ml/runs/mine/r*/gate.json R7 最后一轮左右比例 / 行程门禁
2026-10-18 新增(审查:§9 验收表在流水线里没有执行点)。"""
import argparse, glob, json, os, re
import torch
def jload(p):
try: return json.load(open(p, encoding='utf-8'))
except Exception: return None
def main():
from ml.common.kin_torch import Kin
from ml.model.policy import build
from ml.data import load_tasks
from ml.sim.closed_loop import evaluate
ap = argparse.ArgumentParser()
ap.add_argument('--ckpt', required=True); ap.add_argument('--samples', default='ml/data/samples')
ap.add_argument('--out', default='ml/runs/acceptance.json')
ap.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu')
a = ap.parse_args()
items = []
def add(name, value, ok, rule):
items.append({'item': name, 'value': value, 'pass': (None if ok is None else bool(ok)), 'rule': rule})
# 1) held-out 闭环(仿真安全层)
ck = torch.load(a.ckpt, map_location=a.device)
model = build(ck['cfg']).to(a.device); model.load_state_dict(ck['model']); model.eval()
kin = Kin(device=a.device)
val = [t for t in load_tasks([a.samples]) if t['split'] == 'val']
reach = [t for t in val if t.get('reached') and t['cat'] != 'D']
dock = [t for t in val if t['cat'] == 'D']
if reach:
r, _ = evaluate(model, kin, reach, a.device, safety=True)
add('held-out 可达任务到达率', r['success'], r['success'] >= 0.98, '≥ 0.98(仿真安全层)')
add('起步失败率(R2)', r['start_fail'], r['start_fail'] == 0, '= 0')
else:
add('held-out 可达任务到达率', None, False, 'val 中无可达任务')
if dock:
_, pers = evaluate(model, kin, dock, a.device, safety=True)
err = torch.cat([p['err'] for _, p in pers])
ref = torch.tensor([t.get('err') if t.get('err') is not None else float('inf') for t in dock])
rate = (err <= ref + 5).float().mean().item()
add('D 类停靠距离 ≤ 老师 + 5 mm 的比例', rate, rate >= 0.98, '≥ 0.98')
# 2) 部署形态
pb = jload('ml/runs/policy_bench.json')
if pb is None:
add('部署形态评测', 'missing', False, '先跑 bash ml/pipeline.sh bench')
else:
sets = {k: v for k, v in pb.items() if isinstance(v, dict) and 'n' in v}
cf = sum(v['collisionFrames'] for v in sets.values())
add('部署形态穿模帧(全部集合)', cf, cf == 0, '= 0')
for k in ('near_column_pos', 'near_column_pose', 'route40', 'near_column_1000'):
if k in sets: add(f'{k} 通过', f"{sets[k]['ok']}/{sets[k]['n']}", sets[k]['ok'] == sets[k]['n'], '全部通过')
else: add(f'{k} 通过', 'missing', False, '全部通过')
plans = sum(v.get('plans', 0) for v in sets.values()); br = sum(v.get('brakes', 0) for v in sets.values())
add('刹停发生率(刹停 / 推理)', br / plans if plans else None, (br / plans < 0.02) if plans else False, '< 0.02')
r8 = sets.get('r8_adversarial')
if r8:
add('R8 对抗集 3 s 内恢复率', r8['recoveryRate'], r8['recoveryRate'] >= 0.95, '≥ 0.95')
add('R8 活锁', r8['livelock'], r8['livelock'] == 0, '= 0')
else: add('R8 对抗集', 'missing', False, 'bench 需含 r8')
add('推理延迟 P99(Node CPU,参考)', max((v.get('inferMsP99') or 0) for v in sets.values()), None, '浏览器以 __POLICY.latency() 为准')
# 3) 数据 / 训练 / 后训练门禁
r2 = jload('ml/data/r2.json')
add('R2 近邻标签 std(步长比例)', r2['label_std_mean'] if r2 else 'missing', bool(r2 and r2['pass']), '< 0.10')
sps = None
for f in sorted(glob.glob('ml/runs/bench_*/log.jsonl')):
recs = [json.loads(l) for l in open(f) if '"samples_per_s"' in l]
if recs: sps = max(sps or 0, recs[-1]['samples_per_s'])
add('R10 训练吞吐(样本/s)', sps, sps is not None and sps >= 5000, '≥ 5000(≈ 60000 步 ≤ 3.5 h)')
m5 = [jload(f) for f in sorted(glob.glob('ml/runs/m5_gate_*.json'))]
m5 = [m for m in m5 if m]
add('M5 头门禁', [{'head': m['head'], 'collision_rate': m['policy_only']['collision_rate']} for m in m5] or 'missing',
any(m['pass'] for m in m5), '纯策略贴立柱穿柱率 ≤ 0.02')
rnum = lambda p: int((re.findall(r'[\\/]r(\d+)[\\/]gate\.json$', p) or ['0'])[0])
gates = sorted(glob.glob('ml/runs/mine/r*/gate.json'), key=rnum)
if gates:
gj = jload(gates[-1])
add('R7 左右比例变化(pp)/ 行程比', [gj.get('R7_left_share_shift_pp'), gj.get('R7_travel_median_ratio')], gj.get('R7_ok'), '≤ 5 pp 且 ≤ 1.05')
else: add('R7 门禁', 'missing', False, '先跑 mine')
hard = [i for i in items if i['pass'] is not None]
out = {'ckpt': a.ckpt, 'all_pass': all(i['pass'] for i in hard), 'failed': [i['item'] for i in hard if not i['pass']], 'items': items}
os.makedirs(os.path.dirname(a.out) or '.', exist_ok=True)
json.dump(out, open(a.out, 'w', encoding='utf-8'), indent=1, ensure_ascii=False)
for i in items:
mark = {True: '通过', False: '未过', None: '参考'}[i['pass']]
print(f"[{mark}] {i['item']}: {i['value']} ({i['rule']})")
print('全部通过' if out['all_pass'] else '未通过:' + '、'.join(out['failed']))
if __name__ == '__main__':
main()