第五轮：锁/续跑/超时/自动分支/R4 探针与 encode-once/控制台加固；阶段内断点续跑（gen 按条、prep 按 shard、训练与 APG 定时原子存档+体面停止）；分歧以实现方意见为准

ml/apg/apg_finetune.py

@@ -7,7 +7,7 @@
   "到达"仍是每 seg_plans 段末的距离项（无到达即停），属有意设计：段末项给出持续的接近梯度。
   margin_train 3 mm：与训练数据过滤线一致（认证最小间隙 < 3 mm 的老师轨迹不进训练集），碰撞项不与 BC 标签对抗；
   运行时投影护栏独立保持 5 mm。端点与中点两个铰链取均值，与只查端点同量纲，w_col 无需重标。"""
-import argparse, json, math, os, random, time
+import argparse, json, math, os, random, sys, time
 import torch
 
 from ml.common.kin_torch import Kin
@@ -16,6 +16,7 @@ from ml.data import Samples, load_tasks
 from ml.model.policy import build, wta_loss
 from ml.model.meanflow import meanflow_loss
 from ml.sim.closed_loop import evaluate, tasks_to_tensors, goal_err
+from ml.common.ckpt import Saver, atomic_save
 
 
 def load_pool(paths):
@@ -32,6 +33,7 @@ def main():
     ap.add_argument('--pool', nargs='*', default=[]); ap.add_argument('--out', default='ml/runs/apg')
     ap.add_argument('--config', default='ml/configs/apg.json'); ap.add_argument('--steps', type=int, default=None)
     ap.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu'); ap.add_argument('--seed', type=int, default=0)
+    ap.add_argument('--resume', default=None, help='从 <out>/last.pt 续跑（2026-10-18 第五轮，审查 N-11：几小时的 APG 断了不再从头来）')
     a = ap.parse_args()
     cfg = json.load(open(a.config)); C = cfg['apg']
     if a.steps: C['steps'] = a.steps
@@ -49,13 +51,26 @@ def main():
     opt = torch.optim.AdamW(model.parameters(), lr=C['lr'], betas=(0.9, 0.95), weight_decay=0.0)
     amp = dev.startswith('cuda') and C.get('amp') == 'bf16'
     step = torch.tensor(STEP, device=dev)
-    log = open(os.path.join(a.out, 'log.jsonl'), 'w')
-    base, _ = evaluate(model.eval(), kin, val_tasks, dev)
-    print('EVAL0', json.dumps(base), flush=True); log.write(json.dumps({'eval': base, 'step': 0}) + '\n')
-    best = base['success']
-    torch.save({'model': model.state_dict(), 'cfg': mcfg, 'step': 0, 'fp': kin.fp, 'eval': base}, os.path.join(a.out, 'best.pt'))
+    it0 = 0
+    if a.resume and os.path.exists(a.resume):
+        rk = torch.load(a.resume, map_location=dev); model.load_state_dict(rk['model'])
+        if 'opt' in rk: opt.load_state_dict(rk['opt'])
+        it0 = rk.get('step', 0)
+        bp = os.path.join(a.out, 'best.pt')
+        best = torch.load(bp, map_location='cpu').get('eval', {}).get('success', -1.0) if os.path.exists(bp) else -1.0
+        log = open(os.path.join(a.out, 'log.jsonl'), 'a')
+        print(f'续跑：从第 {it0} 步继续，当前最好 success {best:.4f}', flush=True)
+    else:
+        log = open(os.path.join(a.out, 'log.jsonl'), 'w')
+        base, _ = evaluate(model.eval(), kin, val_tasks, dev)
+        print('EVAL0', json.dumps(base), flush=True); log.write(json.dumps({'eval': base, 'step': 0}) + '\n')
+        best = base['success']
+        atomic_save({'model': model.state_dict(), 'cfg': mcfg, 'step': 0, 'fp': kin.fp, 'eval': base}, os.path.join(a.out, 'best.pt'))
     M = C['margin_train_mm']; t0 = time.time()
-    for it in range(C['steps']):
+    saver = Saver(C.get('ckpt_min', 10))             # 2026-10-18：每 10 分钟 + 收到停止请求时存 last.pt（不再只在每 500 次验证时存）
+    def save_last(n, ev=None):
+        atomic_save({'model': model.state_dict(), 'opt': opt.state_dict(), 'cfg': mcfg, 'step': n, 'fp': kin.fp, 'eval': ev}, os.path.join(a.out, 'last.pt')); saver.mark()
+    for it in range(it0, C['steps']):
         model.train()
         batch = random.choices(train_tasks, weights=w, k=C['batch'])
         q0, T, R, ex, tr = tasks_to_tensors(batch, dev)
@@ -134,14 +149,18 @@ def main():
                'gnorm': gn.item(), **stats, 'min': (time.time() - t0) / 60}
         if it % 10 == 0: print(json.dumps(rec), flush=True)
         log.write(json.dumps(rec) + '\n'); log.flush()
+        if saver.stop():
+            save_last(it + 1); log.close()
+            print(f'收到停止请求：已存档 last.pt（第 {it + 1} 次），下次运行自动续跑', flush=True); sys.exit(143)
+        if saver.due(): save_last(it + 1)
         if (it + 1) % C['eval_every'] == 0 or it == C['steps'] - 1:
             res, _ = evaluate(model.eval(), kin, val_tasks, dev)
             res['it'] = it + 1; res['t'] = time.time()
             print('EVAL', json.dumps(res), flush=True); log.write(json.dumps({'eval': res}) + '\n'); log.flush()
             sd = {'model': model.state_dict(), 'cfg': mcfg, 'step': it + 1, 'fp': kin.fp, 'eval': res}
-            torch.save(sd, os.path.join(a.out, 'last.pt'))
+            save_last(it + 1, res)                                                      # last 带优化器状态供续跑；best 不带（导出/门禁用）
             if res['success'] > best + 1e-4:
-                best = res['success']; torch.save(sd, os.path.join(a.out, 'best.pt'))
+                best = res['success']; atomic_save(sd, os.path.join(a.out, 'best.pt'))
     print(f'done, best success {best:.4f}', flush=True)