diff --git a/docs/探索日志.md b/docs/探索日志.md index 8d2fae1..b81e9ac 100644 --- a/docs/探索日志.md +++ b/docs/探索日志.md @@ -312,6 +312,24 @@ - **结论**:管线已能跑,但**损失尺度失控**(碰撞铰链均值 40 万,远大于 margin 3mm 的合理量级);且 APG 展开是纯策略口径(无反馈/安全层/每步规划),与当前部署形态不一致。**暂不采用**,转「反馈蒸馏」。 - **下一步**:反馈蒸馏——用最终部署配置收集 (特征, 复合动作=模型输出+目标反馈) 样本,微调模型把反馈内化。 +## 轮次 20:反馈蒸馏(exp: `exp/dagger-deploy`,负结论) + +- **假设**:用最终部署配置收集 (特征, 复合动作 = 模型输出 + 目标反馈) 样本,微调后模型可内化反馈(部署时降低/去掉反馈仍能到目标)。 +- **判据**:部署口径下 `fb_gain=0~0.9` 时 success 接近/超过基线(43.8%,n=256),且 time_ok 不降。 +- **配方**:`rollout` 加 collect 钩子(与部署一致:safety、replan='empty'、safety_partial=1、commit_n=1、fb_gain=1.8)收集 500 题 → **17.7 万样本**;`train.py --config distill.json`(lr 1e-4、3000 步、w_value=0、w_sub=0、w_fk=0.1、w_smooth=0.01)+ 主数据 BC 锚定,从 dag2 best 微调。 +- **对照**:dag2 + gain 1.8(n=256:43.8% / 28.9% / 1.6% / 0.216 / 40.0 / 956 ms)。 +- **指标(n=256,部署口径)**: + +| 模型 | gain | 到达 | time_ok_1p3 | 碰撞 | ratio2s | 刹停/题 | 墙钟/题 | +|---|---|---|---|---|---|---|---| +| distill | 0.0 | **0.8%** | — | 0.0% | 0.708 | 2.1 | 762 ms | +| distill | 0.9 | 36.7% | 10.5% | 1.2% | 0.421 | 12.4 | 916 ms | +| distill | 1.8 | 43.4% | 27.7% | 2.0% | 0.228 | 47.4 | 925 ms | +| dag2(基线) | 1.8 | 43.8% | 28.9% | 1.6% | 0.216 | 40.0 | 956 ms | + +- **结论**:**反馈未内化**——`gain=0` 时到达仅 0.8%(与基线纯策略一样);`gain=1.8` 时与基线持平(43.4 vs 43.8,差约 1 题),碰撞/刹停略差。蒸馏样本占比仅 8.5%、标签仍以主数据增量为主,模型没有学会"看目标修正"。此配方不成立。 +- **下一步**:与最终部署配置一致的 DAgger 第 3 轮(采集时已用 safety+fb 1.2,但仍是 3 步提交/全窗口;现在部署是每步规划+部分前缀)。 + ## 分支清理(2026-10-20) - `feat/fb-mix`(8e5431c):反馈混合方案的中间分支,**已验证是 `exp/dagger-deploy` 的祖先**(所有提交都在后者历史里),本地/远端分支删除,内容保留在 exp/dagger-deploy。 diff --git a/ml/configs/distill.json b/ml/configs/distill.json new file mode 100644 index 0000000..70283f0 --- /dev/null +++ b/ml/configs/distill.json @@ -0,0 +1,41 @@ +{ + "model": { + "d": 192, + "heads": 8, + "ff": 768, + "enc": 4, + "dec": 4 + }, + "train": { + "batch": 1024, + "lr": 0.0001, + "warmup": 1000, + "steps": 3000, + "wd": 0.01, + "betas": [ + 0.9, + 0.95 + ], + "amp": "bf16", + "grad_clip": 1.0, + "w_mf": 1.0, + "w_fk": 0.1, + "w_smooth": 0.01, + "w_value": 0.0, + "w_sub": 0.0, + "p_neq": 0.5, + "p_edge": 0.1, + "p_neq_warm": 0, + "fk_tmax": 0.3, + "fk_caps": 96, + "ema": 0.999, + "aug_pre_drop": 0.15, + "aug_pre_sigma": 0.1, + "aug_dq_sigma": 0.3, + "ckpt_min": 10, + "eval_every": 1000, + "eval_tasks": 256, + "log_every": 50, + "patience_evals": 3 + } +} \ No newline at end of file diff --git a/ml/dagger/distill_fb.py b/ml/dagger/distill_fb.py new file mode 100644 index 0000000..6c8c5d1 --- /dev/null +++ b/ml/dagger/distill_fb.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +"""反馈蒸馏:用最终部署配置(目标反馈 + 安全层 + 部分前缀 + 每步规划)跑闭环, +收集 (特征, 复合动作 = 模型输出 + 目标反馈) 样本,供 train.py 微调把反馈内化进策略。 + PYTHONUTF8=1 python -m ml.dagger.distill_fb --ckpt ml/runs/base_dag2_dag1/best.pt --n 500 \ + --fb_gain 1.8 --out ml/data/samples_fbdistill +输出可直接被 ml.data.Samples 读取(collect.X/Y/V/SG/TID.npy;无 meta = act_scale 1.0,与主数据一致)。 +动作标签 = 候选动作(含反馈)的归一化 act(/STEP,clamp ±1),与主数据同口径。 +""" +import argparse, os, random +import numpy as np +import torch + +from ml.common.ckpt import load_policy +from ml.common.features import STEP +from ml.common.kin_torch import Kin +from ml.data import load_tasks +from ml.sim.closed_loop import rollout + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--ckpt', required=True); ap.add_argument('--n', type=int, default=500) + ap.add_argument('--bs', type=int, default=64); ap.add_argument('--fb_gain', type=float, default=1.8) + ap.add_argument('--samples', default='ml/data/samples'); ap.add_argument('--out', required=True) + ap.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu'); ap.add_argument('--seed', type=int, default=1) + a = ap.parse_args() + dev = a.device + model, _ = load_policy(a.ckpt, dev) + kin = Kin(device=dev) + pool = [t for t in load_tasks([a.samples]) if t['split'] == 'train' and t.get('cat') not in ('D',)] + random.Random(a.seed).shuffle(pool); tasks = pool[:a.n] + stp = torch.tensor(STEP) # collect 返回 CPU 张量,标签换算也在 CPU + X, Y, TID = [], [], [] + for i in range(0, len(tasks), a.bs): + ch = tasks[i:i + a.bs] + coll = [] + with torch.no_grad(): + rollout(model, kin, ch, dev, K=8, safety=True, replan='empty', fb_gain=a.fb_gain, + safety_partial=1, commit_n=1, seed=a.seed * 100003 + i, collect=coll) + for f, ac in coll: + X.append(f.numpy()); Y.append((ac / stp).clamp(-1, 1).numpy()) + ncol = sum(f.shape[0] for f, _ in coll) + TID.append(np.arange(ncol, dtype=np.int32) + i * 100000) + print(f'[distill] {i + len(ch)}/{len(tasks)} tasks, samples {sum(x.shape[0] for x in X)}', flush=True) + os.makedirs(a.out, exist_ok=True) + X = np.concatenate(X); Y = np.concatenate(Y); TID = np.concatenate(TID) + n = X.shape[0] + np.save(os.path.join(a.out, 'collect.X.npy'), X.astype(np.float32)) + np.save(os.path.join(a.out, 'collect.Y.npy'), Y.astype(np.float32)) + np.save(os.path.join(a.out, 'collect.V.npy'), np.full(n, 0.5, np.float32)) + np.save(os.path.join(a.out, 'collect.SG.npy'), np.zeros((n, 13), np.float32)) + np.save(os.path.join(a.out, 'collect.TID.npy'), TID) + print(f'done: {n} samples, X {X.shape}, Y {Y.shape}', flush=True) + + +if __name__ == '__main__': + main() diff --git a/ml/sim/closed_loop.py b/ml/sim/closed_loop.py index 989142a..f66427f 100644 --- a/ml/sim/closed_loop.py +++ b/ml/sim/closed_loop.py @@ -142,7 +142,7 @@ class TeacherOracle: @torch.no_grad() def rollout(model, kin, tasks, device, K=8, safety=False, seed=0, max_sec=None, return_traj=False, use_sub=True, sub_override=None, qp0=None, steps=1, z_scale=1.0, select='consist', oracle=None, replan='r3', sub_rule='cross', - fb_gain=0.0, fb_scale=0.0, fb_near=0.2, fb_fallback=0.0, safety_partial=0, commit_n=3): + fb_gain=0.0, fb_scale=0.0, fb_near=0.2, fb_fallback=0.0, safety_partial=0, commit_n=3, collect=None): """use_sub=False:不用子目标(sf=0);sub_override=(subs [B,M,6] 绝对关节角, n [B] 个数 或 need [B] bool):冷启动时用给定途经点代替子目标头; qp0 [B,6]:起点的上一帧(默认 = q0,即静止起步)。steps:MeanFlow 采样段数;z_scale:噪声缩放; select:consist(部署口径:冷启动价值前瞻、之后与上一条已提交计划的一致性)/ value(每次都价值前瞻)/ first(不选,取第 0 条)。 @@ -234,6 +234,8 @@ def rollout(model, kin, tasks, device, K=8, safety=False, seed=0, max_sec=None, order = torch.sort(score, dim=-1, descending=True, stable=True).indices pick = order[:, 0] commit = inc[ar, pick, r:r + commit_n] + if collect is not None: # 反馈蒸馏:收集与部署完全一致的 (特征, 复合动作=模型输出+目标反馈) + collect.append((feat.detach().float().cpu(), inc[ar, pick].detach().float().cpu())) committed = torch.ones(B, dtype=torch.bool, device=device) if safety and safety_partial: # 2026-10-20:允许「部分安全前缀」——执行到第一个不安全步为止,而不是全拒刹停 maxlen = torch.zeros(B, dtype=torch.long, device=device)