这是示例数据，不是你的训练。${location.protocol === 'file:' ? '直接双击 monitor.html 只能看示例；' : ''}看真实训练请双击 ml/console.bat（或 python -m ml.monitor --open）。
`); }
+ if (DEMO) { setPill('demo', '示例数据'); if (!$('demobar')) $('pipe').closest('.hero').insertAdjacentHTML('beforebegin', `这是示例数据，不是你的训练。${location.protocol === 'file:' ? '直接双击 monitor.html 只能看示例；' : ''}看真实训练请双击 ml/console.bat（或 python -m ml.monitor --open）。
`); }
else if (busy) setPill('live', `运行中 · ${P && P.current ? SM[P.current]?.n || P.current : '启动中'}`);
else if (cur && Date.now() / 1000 - lastMtime < 120) setPill('live', `运行中 · ${ago(lastMtime)}更新`);
else setPill('', '空闲');
@@ -531,15 +533,16 @@ function renderStatus(s) {
// GPU
const G = s.gpu || [];
+ const BUD = (s.vram_budget_gb || 24) * 1024; // 显存预算（MB）：桌面常驻占 4–5 GB，不能按整卡 24 GB 看
$('gputime').textContent = G.length ? '实时' : '';
$('gpu').innerHTML = G.length ? G.map(g => {
const u = g.util ?? 0, C = 2 * Math.PI * 50, col = u > 85 ? css('--ok') : u > 50 ? css('--accent') : css('--warn');
- const memP = 100 * g.mem_used / g.mem_total, tc = g.temp > 83 ? css('--bad') : g.temp > 75 ? css('--warn') : css('--c6');
+ const memP = 100 * g.mem_used / BUD, tc = g.temp > 83 ? css('--bad') : g.temp > 75 ? css('--warn') : css('--c6');
const row = (k, v, p, c) => `
-
${row('显存', `${(g.mem_used / 1024).toFixed(1)} / ${(g.mem_total / 1024).toFixed(0)} GB`, memP, css('--accent2'))}
+
${row('显存', `${(g.mem_used / 1024).toFixed(1)} / ${(BUD / 1024).toFixed(0)} GB 预算（整卡 ${(g.mem_total / 1024).toFixed(0)}）`, memP, memP > 100 ? css('--bad') : memP > 85 ? css('--warn') : css('--accent2'))}
${row('温度', `${g.temp} °C`, g.temp, tc)}${row('功耗', g.power != null ? `${Math.round(g.power)} W` : '—', g.power ? g.power / 3.5 : 0, css('--c7'))}
${g.name}
`;
}).join('
') : empty('未检测到 nvidia-smi');
diff --git a/ml/monitor.py b/ml/monitor.py
index 17d543d..40f2e80 100644
--- a/ml/monitor.py
+++ b/ml/monitor.py
@@ -192,7 +192,8 @@ def status():
pb = jload(P(RUNS, 'policy_bench.json'))
if pb: pb = {k: {kk: v.get(kk) for kk in ('n', 'ok', 'reached', 'collisionFrames', 'brakeRate', 'recoveryRate', 'livelock', 'inferMsP99', 'safetyMsP99')}
for k, v in pb.items() if isinstance(v, dict) and 'n' in v}
- return {'now': time.time(), 'gpu': gpu(), 'pipeline': pl, 'check': CHECK.get(), 'lock': lock_info(),
+ return {'now': time.time(), 'gpu': gpu(),
+ 'vram_budget_gb': float(os.environ.get('VRAM_BUDGET_GB', 20)), # 本机桌面常驻占 4–5 GB，给训练的预算 19–20 GB 'pipeline': pl, 'check': CHECK.get(), 'lock': lock_info(),
'stage_meta': [{'id': k, 'name': n, 'est_h': h, 'timeout_h': pstate.TIMEOUT_H[k]} for k, n, h in pstate.STAGES],
'r4': jload(P(RUNS, 'r4.json')),
'data': {'consistency': (jload(P(DATA, 'consistency.json')) or {}).get('consistentByGateRule'),
@@ -218,8 +219,13 @@ def find_bash():
if p and os.path.exists(p): return p
g = shutil.which('git')
if g:
- p = os.path.join(os.path.dirname(os.path.dirname(g)), 'bin', 'bash.exe')
- if os.path.exists(p): return p
+ root = os.path.dirname(os.path.dirname(g)) # 便携版 git 常放在 <根>\mingw64\bin\git.exe，要再上一层才找到 bash
+ for base in dict.fromkeys([root, os.path.dirname(root)]):
+ for sub in ('bin', os.path.join('usr', 'bin')):
+ p = os.path.join(base, sub, 'bash.exe')
+ if os.path.exists(p): return p
+ b = shutil.which('bash')
+ if b and 'windows\\system32' not in b.lower(): return b
return None
return shutil.which('bash')
diff --git a/ml/selftest.sh b/ml/selftest.sh
index 9e90000..432438a 100755
--- a/ml/selftest.sh
+++ b/ml/selftest.sh
@@ -17,14 +17,21 @@ git ls-files -z | tar --null -T - -cf - | (cd "$D" && tar xf -)
[ -d ml/node_modules ] && cp -r ml/node_modules "$D/ml/node_modules"
cd "$D"; mkdir -p ml/runs ml/data
R=ml/runs/selftest_items.jsonl; : > $R
-rec() { $PY -c "import json,sys;print(json.dumps({'item':sys.argv[1],'pass':json.loads(sys.argv[2]),'detail':sys.argv[3]},ensure_ascii=False))" "$1" "$2" "$3" >> $R; echo "[selftest] $1 → $2 $3"; }
+rec() { $PY -c "import json,sys
+try: v=json.loads(sys.argv[2])
+except Exception: v=False # 值不是合法 JSON 时记 false，不能让整项静默消失（第七轮实测：'none' 抛异常 → 两项没进结果）
+print(json.dumps({'item':sys.argv[1],'pass':v,'detail':sys.argv[3]},ensure_ascii=False))" "$1" "$2" "$3" >> $R; echo "[selftest] $1 → $2 $3"; }
CFGS=ml/configs/smoke.json
# 1 生成续写
node ml/gen/gen_dataset.cjs --n 6 --seed 11 --out ml/data/a.jsonl 2>/dev/null
node ml/gen/gen_dataset.cjs --n 3 --seed 11 --out ml/data/shard_0011.jsonl.part 2>/dev/null
sz=$(wc -c < ml/data/shard_0011.jsonl.part); head -c $((sz - 40)) ml/data/shard_0011.jsonl.part > t && mv t ml/data/shard_0011.jsonl.part
node ml/gen/gen_dataset.cjs --n 6 --seed 11 --out ml/data/shard_0011.jsonl.part 2>ml/runs/st_gen.log; mv ml/data/shard_0011.jsonl.part ml/data/shard_0011.jsonl
-cmp -s ml/data/a.jsonl ml/data/shard_0011.jsonl && rec gen_resume true "$(grep -o '续写[^,]*' ml/runs/st_gen.log | head -1)" || rec gen_resume false "续写结果与一口气生成不同"
+gd=$($PY -c "import json,sys
+f=lambda p:[json.dumps({**r,'plan_ms':0,'cert':{**r.get('cert',{}),'ms':0}},sort_keys=True) for r in (json.loads(l) for l in open(p,encoding='utf-8') if l.strip())]
+a,b=f(sys.argv[1]),f(sys.argv[2]);print('OK' if a==b else '不同 %d/%d 条'%(sum(x!=y for x,y in zip(a,b))+abs(len(a)-len(b)),max(len(a),len(b))))" ml/data/a.jsonl ml/data/shard_0011.jsonl)
+if [ "$gd" = OK ]; then rec gen_resume true "$(grep -o '续写[^,]*' ml/runs/st_gen.log | head -1)（逐条一致；不含 plan_ms/cert.ms 两个墙钟字段，第七轮实测 cmp -s 恒不等）"
+else rec gen_resume false "续写与一口气生成不同：$gd"; fi
rm -f ml/data/a.jsonl
node ml/gen/gen_dataset.cjs --n 40 --seed 12 --out ml/data/shard_0012.jsonl 2>/dev/null # 训练自检用的小数据
# 2 打包二次运行
@@ -49,13 +56,16 @@ if command -v timeout >/dev/null; then
grep -q "收到停止请求" ml/runs/st_c.log && rec train_term true "rc=$rc，TERM 送达并存档" || rec train_term false "rc=$rc，TERM 没送到 python（Windows 原生进程常见）→ 阶段超时只能硬杀，进度靠每 10 分钟存档"
else rec train_term false "没有 timeout 命令"; fi
# 6 APG 续跑
-$PY -m ml.apg.apg_finetune --init ml/runs/st_a/last.pt --samples ml/data/samples --out ml/runs/st_apg --steps 10 --stop-at 5 > ml/runs/st_apg.log 2>&1; rc=$?
-$PY -m ml.apg.apg_finetune --init ml/runs/st_a/last.pt --samples ml/data/samples --out ml/runs/st_apg --steps 10 --resume ml/runs/st_apg/last.pt >> ml/runs/st_apg.log 2>&1; rc2=$?
+$PY -m ml.apg.apg_finetune --config $CFGS --init ml/runs/st_a/last.pt --samples ml/data/samples --out ml/runs/st_apg --steps 10 --stop-at 5 > ml/runs/st_apg.log 2>&1; rc=$?
+$PY -m ml.apg.apg_finetune --config $CFGS --init ml/runs/st_a/last.pt --samples ml/data/samples --out ml/runs/st_apg --steps 10 --resume ml/runs/st_apg/last.pt >> ml/runs/st_apg.log 2>&1; rc2=$?
[ $rc = 143 ] && [ $rc2 = 0 ] && rec apg_resume true "" || rec apg_resume false "rc=$rc 续跑rc=$rc2（日志 $D/ml/runs/st_apg.log）"
# 7 进程树 / 关窗口
pj=$($PY ml/selftest_proc.py 2>ml/runs/st_proc.err | tail -1)
echo "$pj" > ml/runs/st_proc.json
-k=$($PY -c "import json,sys;d=json.loads(sys.argv[1]);print(str(d.get('tree_killed')).lower(), str(d.get('survive_parent_exit')).lower())" "$pj" 2>/dev/null || echo "null null")
+k=$($PY -c "import json,sys
+try: d=json.loads(sys.argv[1])
+except Exception: d={}
+print(json.dumps(d.get('tree_killed')), json.dumps(d.get('survive_parent_exit')))" "$pj" 2>/dev/null || echo "null null")
rec tree_killed ${k% *} "$pj" ; rec survive_parent_exit ${k#* } ""
mkdir -p "$SRC/ml/runs"
$PY - "$R" "$SRC/ml/runs/selftest.json" "$D" <<'PY'
diff --git a/ml/selftest_proc.py b/ml/selftest_proc.py
index 0724988..0224fbd 100644
--- a/ml/selftest_proc.py
+++ b/ml/selftest_proc.py
@@ -14,8 +14,9 @@ FLAGS = {'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP | 0x08000000} if o
def alive(pid):
if os.name == 'nt':
- r = subprocess.run(['tasklist', '/FI', f'PID eq {pid}', '/NH'], capture_output=True, text=True)
- return str(pid) in r.stdout
+ # 中文 Windows 的 tasklist 表头是 GBK，text=True 会让读线程解码崩掉（stdout 变 None）→ 按字节读再容错解码
+ r = subprocess.run(['tasklist', '/FI', f'PID eq {pid}', '/NH'], capture_output=True, text=True, errors='replace')
+ return str(pid) in (r.stdout or '')
try: os.kill(pid, 0); return True
except OSError: return False
@@ -26,11 +27,14 @@ def tree_cmd(d):
f'node -e "require(\'fs\').writeFileSync(\'{dd}/n.pid\',String(process.pid));setInterval(()=>{{}},1000)" & wait')
+WAITERR = ['']
+
+
def wait_pids(d, t=20):
for _ in range(t * 4):
try: return int(open(os.path.join(d, 'p.pid')).read()), int(open(os.path.join(d, 'n.pid')).read())
- except Exception: time.sleep(0.25)
- return None, None
+ except Exception as e: WAITERR[0] = type(e).__name__ + ': ' + str(e)[:90]; time.sleep(0.25)
+ return None, None # 起不来时把最后一次异常带进结果，别让 tree_started=false 变成没有原因的失败
def main():
diff --git a/ml/train.py b/ml/train.py
index fcfab9a..2706eab 100644
--- a/ml/train.py
+++ b/ml/train.py
@@ -9,6 +9,7 @@ import torch
from ml.common.kin_torch import Kin
from ml.common.features import STEP, D2R
from ml.common.ckpt import Saver, atomic_save, rng_state, set_rng_state
+from ml.common import vram
from ml.data import Samples, load_tasks
from ml.model.policy import build, wta_loss
from ml.model.meanflow import meanflow_loss, one_step
@@ -40,6 +41,7 @@ def main():
os.makedirs(a.out, exist_ok=True)
torch.manual_seed(a.seed)
dev = a.device
+ vram.limit()
kin = Kin(device=dev)
data = Samples(a.samples, device=dev, split='train')
val_tasks = [t for t in load_tasks(a.samples) if t['split'] == 'val'][:tc['eval_tasks']]
@@ -111,7 +113,7 @@ def main():
el = time.time() - t0
rec = {'step': step, 'total': total, 't': time.time(), 'loss': loss.item(), 'mf': mf_raw.item(), 'fk_cm': l_fk.item(), 'smooth': l_sm.item(),
'value': l_val.item(), 'sub': l_sub.item(), 'gnorm': gn.item(), 'lr': lr_at(step), 'samples_per_s': seen / max(el, 1e-6),
- 'eta_h': (total - step - 1) * tc['batch'] / max(seen / max(el, 1e-6), 1e-6) / 3600}
+ 'eta_h': (total - step - 1) * tc['batch'] / max(seen / max(el, 1e-6), 1e-6) / 3600, 'vram': vram.peak_gib()}
print(json.dumps(rec), flush=True); log.write(json.dumps(rec) + '\n'); log.flush()
if not a.no_eval and (step + 1) % tc['eval_every'] == 0:
model.eval()