ml: 按运行手册审查修订——头分目录不互相覆盖(ml/runs/<CFG>、selected_ckpt/CKPT);统一 $PYTHON(Windows Git Bash 可跑);gen 按实测样本/任务自动补量;R10 改终态判据(≥5k 样本/s≈60k 步≤3.5h);m5 纯策略/仿真安全层双口径;无老师题定长 6 s 口径;APG 展开随批时长、端点+中点铰链、余量 3 mm(投影同步);挖错池只用本轮;门禁加 R7;bench 加 R8 对抗集与刹停率、--nc-n/--route-n/--r8-n;新增 check_r2 与 acceptance;export 默认 fp32;出题分块防 OOM;val 为空保护
This commit is contained in:
@@ -0,0 +1,104 @@
|
||||
"""验收汇总(方案 §9.2 + §10.1 R2/R7/R8/R10):把各阶段产物里的判据集中核对,输出 ml/runs/acceptance.json。
|
||||
python -m ml.eval.acceptance --ckpt ml/runs/current.pt --samples ml/data/samples --out ml/runs/acceptance.json
|
||||
自己跑的一项:held-out(val 划分)全部任务闭环仿真(开仿真安全层)——可达任务到达率 ≥ 98%、D 类停靠距离 ≤ 老师 + 5 mm、起步失败 = 0。
|
||||
其余项读已有文件(缺文件记为 missing,不算通过):
|
||||
ml/runs/policy_bench.json 部署形态:各集合穿模帧 = 0;near 30/30(pos、pose);route 全部到达;nc1000 0 失败;刹停率 < 2%;R8 恢复 ≥ 95%、活锁 = 0
|
||||
ml/data/r2.json R2 近邻标签 std < 10% 步长
|
||||
ml/runs/bench_*/log.jsonl R10 吞吐 ≥ 5000 样本/s(≈ 60000 步 ≤ 3.5 h)
|
||||
ml/runs/m5_gate_*.json M5 头门禁
|
||||
ml/runs/mine/r*/gate.json R7 最后一轮左右比例 / 行程门禁
|
||||
2026-10-18 新增(审查:§9 验收表在流水线里没有执行点)。"""
|
||||
import argparse, glob, json, os, re
|
||||
import torch
|
||||
|
||||
|
||||
def jload(p):
|
||||
try: return json.load(open(p, encoding='utf-8'))
|
||||
except Exception: return None
|
||||
|
||||
|
||||
def main():
|
||||
from ml.common.kin_torch import Kin
|
||||
from ml.model.policy import build
|
||||
from ml.data import load_tasks
|
||||
from ml.sim.closed_loop import evaluate
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--ckpt', required=True); ap.add_argument('--samples', default='ml/data/samples')
|
||||
ap.add_argument('--out', default='ml/runs/acceptance.json')
|
||||
ap.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu')
|
||||
a = ap.parse_args()
|
||||
items = []
|
||||
def add(name, value, ok, rule):
|
||||
items.append({'item': name, 'value': value, 'pass': (None if ok is None else bool(ok)), 'rule': rule})
|
||||
|
||||
# 1) held-out 闭环(仿真安全层)
|
||||
ck = torch.load(a.ckpt, map_location=a.device)
|
||||
model = build(ck['cfg']).to(a.device); model.load_state_dict(ck['model']); model.eval()
|
||||
kin = Kin(device=a.device)
|
||||
val = [t for t in load_tasks([a.samples]) if t['split'] == 'val']
|
||||
reach = [t for t in val if t.get('reached') and t['cat'] != 'D']
|
||||
dock = [t for t in val if t['cat'] == 'D']
|
||||
if reach:
|
||||
r, _ = evaluate(model, kin, reach, a.device, safety=True)
|
||||
add('held-out 可达任务到达率', r['success'], r['success'] >= 0.98, '≥ 0.98(仿真安全层)')
|
||||
add('起步失败率(R2)', r['start_fail'], r['start_fail'] == 0, '= 0')
|
||||
else:
|
||||
add('held-out 可达任务到达率', None, False, 'val 中无可达任务')
|
||||
if dock:
|
||||
_, pers = evaluate(model, kin, dock, a.device, safety=True)
|
||||
err = torch.cat([p['err'] for _, p in pers])
|
||||
ref = torch.tensor([t.get('err') if t.get('err') is not None else float('inf') for t in dock])
|
||||
rate = (err <= ref + 5).float().mean().item()
|
||||
add('D 类停靠距离 ≤ 老师 + 5 mm 的比例', rate, rate >= 0.98, '≥ 0.98')
|
||||
|
||||
# 2) 部署形态
|
||||
pb = jload('ml/runs/policy_bench.json')
|
||||
if pb is None:
|
||||
add('部署形态评测', 'missing', False, '先跑 bash ml/pipeline.sh bench')
|
||||
else:
|
||||
sets = {k: v for k, v in pb.items() if isinstance(v, dict) and 'n' in v}
|
||||
cf = sum(v['collisionFrames'] for v in sets.values())
|
||||
add('部署形态穿模帧(全部集合)', cf, cf == 0, '= 0')
|
||||
for k in ('near_column_pos', 'near_column_pose', 'route40', 'near_column_1000'):
|
||||
if k in sets: add(f'{k} 通过', f"{sets[k]['ok']}/{sets[k]['n']}", sets[k]['ok'] == sets[k]['n'], '全部通过')
|
||||
else: add(f'{k} 通过', 'missing', False, '全部通过')
|
||||
plans = sum(v.get('plans', 0) for v in sets.values()); br = sum(v.get('brakes', 0) for v in sets.values())
|
||||
add('刹停发生率(刹停 / 推理)', br / plans if plans else None, (br / plans < 0.02) if plans else False, '< 0.02')
|
||||
r8 = sets.get('r8_adversarial')
|
||||
if r8:
|
||||
add('R8 对抗集 3 s 内恢复率', r8['recoveryRate'], r8['recoveryRate'] >= 0.95, '≥ 0.95')
|
||||
add('R8 活锁', r8['livelock'], r8['livelock'] == 0, '= 0')
|
||||
else: add('R8 对抗集', 'missing', False, 'bench 需含 r8')
|
||||
add('推理延迟 P99(Node CPU,参考)', max((v.get('inferMsP99') or 0) for v in sets.values()), None, '浏览器以 __POLICY.latency() 为准')
|
||||
|
||||
# 3) 数据 / 训练 / 后训练门禁
|
||||
r2 = jload('ml/data/r2.json')
|
||||
add('R2 近邻标签 std(步长比例)', r2['label_std_mean'] if r2 else 'missing', bool(r2 and r2['pass']), '< 0.10')
|
||||
sps = None
|
||||
for f in sorted(glob.glob('ml/runs/bench_*/log.jsonl')):
|
||||
recs = [json.loads(l) for l in open(f) if '"samples_per_s"' in l]
|
||||
if recs: sps = max(sps or 0, recs[-1]['samples_per_s'])
|
||||
add('R10 训练吞吐(样本/s)', sps, sps is not None and sps >= 5000, '≥ 5000(≈ 60000 步 ≤ 3.5 h)')
|
||||
m5 = [jload(f) for f in sorted(glob.glob('ml/runs/m5_gate_*.json'))]
|
||||
m5 = [m for m in m5 if m]
|
||||
add('M5 头门禁', [{'head': m['head'], 'collision_rate': m['policy_only']['collision_rate']} for m in m5] or 'missing',
|
||||
any(m['pass'] for m in m5), '纯策略贴立柱穿柱率 ≤ 0.02')
|
||||
rnum = lambda p: int((re.findall(r'[\\/]r(\d+)[\\/]gate\.json$', p) or ['0'])[0])
|
||||
gates = sorted(glob.glob('ml/runs/mine/r*/gate.json'), key=rnum)
|
||||
if gates:
|
||||
gj = jload(gates[-1])
|
||||
add('R7 左右比例变化(pp)/ 行程比', [gj.get('R7_left_share_shift_pp'), gj.get('R7_travel_median_ratio')], gj.get('R7_ok'), '≤ 5 pp 且 ≤ 1.05')
|
||||
else: add('R7 门禁', 'missing', False, '先跑 mine')
|
||||
|
||||
hard = [i for i in items if i['pass'] is not None]
|
||||
out = {'ckpt': a.ckpt, 'all_pass': all(i['pass'] for i in hard), 'failed': [i['item'] for i in hard if not i['pass']], 'items': items}
|
||||
os.makedirs(os.path.dirname(a.out) or '.', exist_ok=True)
|
||||
json.dump(out, open(a.out, 'w', encoding='utf-8'), indent=1, ensure_ascii=False)
|
||||
for i in items:
|
||||
mark = {True: '通过', False: '未过', None: '参考'}[i['pass']]
|
||||
print(f"[{mark}] {i['item']}: {i['value']} ({i['rule']})")
|
||||
print('全部通过' if out['all_pass'] else '未通过:' + '、'.join(out['failed']))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user