diff --git a/ml/gen/run_parallel.sh b/ml/gen/run_parallel.sh
index cedcc47..59cb8a7 100755
--- a/ml/gen/run_parallel.sh
+++ b/ml/gen/run_parallel.sh
@@ -29,9 +29,21 @@ while :; do
echo "[gen 节拍] $(( (tl-t0)/60 )) 分钟 运行中 $alive/${#pids[@]} 进程 本批已写 $n 条 $last"
fi
done
+# 判定按产物而不是 wait 的退出码：2026-10-18 实测 28 个种子跑了 5 小时后全部 wait 报失败，
+# 但每个 .part 都是完整的 1786 行（node 打了 done、mv 却没执行）——Git Bash 长跑后 wait 的返回码不可靠。
fail=0
for k in "${!pids[@]}"; do
- if ! wait ${pids[$k]}; then echo "[gen] 种子 ${seeds[$k]} 失败，最后几行："; tail -n 5 ml/data/log_${seeds[$k]}.txt || true; fail=1; fi
+ s=${seeds[$k]}; f=ml/data/shard_$(printf %04d $s).jsonl
+ wait ${pids[$k]} || true
+ if [ -f "$f" ]; then continue; fi
+ if [ -f "$f.part" ]; then
+ n=$(wc -l < "$f.part")
+ if [ "$n" -ge "$N" ]; then mv "$f.part" "$f" && echo "[gen] 种子 $s 产物完整（$n 行），补改名" && continue; fi
+ echo "[gen] 种子 $s 失败：.part 只有 $n/$N 行，最后几行："; tail -n 5 ml/data/log_$s.txt || true
+ else
+ echo "[gen] 种子 $s 失败：没有产物，最后几行："; tail -n 5 ml/data/log_$s.txt || true
+ fi
+ fail=1
done
[ $fail = 0 ] || exit 6
node ml/gen/stats.cjs ml/data/shard_*.jsonl > ml/data/stats.json
diff --git a/ml/monitor.html b/ml/monitor.html
index d078abf..3beb255 100644
--- a/ml/monitor.html
+++ b/ml/monitor.html
@@ -161,7 +161,7 @@
-
+
@@ -464,10 +464,12 @@ async function loadRuns() {
if (!r) { cur = null; $('charts').innerHTML = ''; $('kpis').innerHTML = ''; $('prog').innerHTML = '当前运行 · 训练曲线
' + empty('还没有训练日志：预训练开始后这里出现损失、吞吐和评估曲线。'); }
}
+let polling = false;
async function poll() {
+ if (polling) return; polling = true; // 1 秒一轮：上一轮没回来就跳过，避免请求堆叠
try {
- lastStatus = await api('/api/status'); applyStageMeta(lastStatus.stage_meta);
- job = await api('/api/job').catch(() => null);
+ const [st, jb] = await Promise.all([api('/api/status'), api('/api/job').catch(() => null)]);
+ lastStatus = st; applyStageMeta(st.stage_meta); job = jb;
await loadRuns();
if (cur) {
const d = await api(`/api/log?run=${encodeURIComponent(cur.name)}&offset=${offset}`);
@@ -487,7 +489,7 @@ async function poll() {
await pollConsole();
} catch (err) {
setPill('dead', '连不上控制台（双击 ml/console.bat 或 python -m ml.monitor）'); console.error(err);
- }
+ } finally { polling = false; }
}
function setPill(cls, text) { const st = $('state'); st.className = 'pill ' + cls; st.lastElementChild.textContent = text; }
@@ -674,8 +676,8 @@ $('smooth').oninput = () => { $('smoothv').textContent = smoothv().toFixed(2); r
$('logy').onclick = () => { logyOn = !logyOn; $('logy').classList.toggle('on', logyOn); render(); };
$('run').onchange = () => { userPicked = true; cur = null; poll(); };
$('scenario').onchange = () => { api.scenario($('scenario').value); cur = null; userPicked = false; conOff = 0; $('run').innerHTML = ''; poll(); };
-$('auto').onclick = () => { const on = !$('auto').classList.contains('on'); $('auto').classList.toggle('on', on); clearInterval(timer); if (on) timer = setInterval(poll, 5000); };
-timer = setInterval(poll, 5000); poll();
+$('auto').onclick = () => { const on = !$('auto').classList.contains('on'); $('auto').classList.toggle('on', on); clearInterval(timer); if (on) timer = setInterval(poll, 1000); };
+timer = setInterval(poll, 1000); poll();
/* =====================================================================
示例数据：两种场景，每次刷新让"运行中"的那条日志向前推进。
diff --git a/ml/monitor.py b/ml/monitor.py
index f035bcf..f03ee89 100644
--- a/ml/monitor.py
+++ b/ml/monitor.py
@@ -68,7 +68,12 @@ def read_log(name, offset):
return {'records': recs, 'offset': offset + end, 'size': size, 'reset': offset == 0, 'mtime': os.path.getmtime(f)}
+_GPU_CACHE = {'t': 0.0, 'v': []}
+
+
def gpu():
+ # 网页 1 秒轮询：nvidia-smi 每次要 150–600 ms，缓存 2 秒（GPU 数字不需要更快），避免每轮都起子进程
+ if time.time() - _GPU_CACHE['t'] < 2.0: return _GPU_CACHE['v']
try:
q = 'name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw'
r = subprocess.run(['nvidia-smi', f'--query-gpu={q}', '--format=csv,noheader,nounits'], capture_output=True, text=True, timeout=3)
@@ -77,9 +82,10 @@ def gpu():
n, u, mu, mt, t, p = [x.strip() for x in line.split(',')]
num = lambda x: float(x) if re.fullmatch(r'[\d.]+', x) else None
out.append({'name': n, 'util': num(u), 'mem_used': num(mu), 'mem_total': num(mt), 'temp': num(t), 'power': num(p)})
- return out
except Exception:
- return []
+ out = []
+ _GPU_CACHE.update(t=time.time(), v=out)
+ return out
def last_rec(name):