From c128288c1ba9891dcca7a6bc509dbebecd4fb30d Mon Sep 17 00:00:00 2001
From: cbb <2625274797@qq.com>
Date: Sun, 18 Oct 2026 13:48:30 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20gen=20=E5=88=A4=E5=AE=9A=E6=94=B9?=
 =?UTF-8?q?=E6=8C=89=E4=BA=A7=E7=89=A9=E8=80=8C=E4=B8=8D=E6=98=AF=20wait?=
 =?UTF-8?q?=20=E9=80=80=E5=87=BA=E7=A0=81=E2=80=94=E2=80=94=E5=AE=9E?=
 =?UTF-8?q?=E6=B5=8B=2028=20=E4=B8=AA=E7=A7=8D=E5=AD=90=E8=B7=91=205=20?=
 =?UTF-8?q?=E5=B0=8F=E6=97=B6=E5=90=8E=E5=85=A8=E9=83=A8=20wait=20?=
 =?UTF-8?q?=E6=8A=A5=E5=A4=B1=E8=B4=A5=EF=BC=8C=E4=BD=86=E6=AF=8F=E4=B8=AA?=
 =?UTF-8?q?=20.part=20=E9=83=BD=E6=98=AF=E5=AE=8C=E6=95=B4=E7=9A=84=201786?=
 =?UTF-8?q?=20=E8=A1=8C=EF=BC=88node=20=E6=89=93=E4=BA=86=20done=E3=80=81m?=
 =?UTF-8?q?v=20=E6=B2=A1=E6=89=A7=E8=A1=8C=EF=BC=89=EF=BC=8CGit=20Bash=20?=
 =?UTF-8?q?=E9=95=BF=E8=B7=91=E5=90=8E=20wait=20=E8=BF=94=E5=9B=9E?=
 =?UTF-8?q?=E7=A0=81=E4=B8=8D=E5=8F=AF=E9=9D=A0=EF=BC=9B=E7=8E=B0=E5=9C=A8?=
 =?UTF-8?q?=E5=8F=AA=E8=A6=81=20.part=20=E8=A1=8C=E6=95=B0=E8=BE=BE?=
 =?UTF-8?q?=E6=A0=87=E5=B0=B1=E8=A1=A5=E6=94=B9=E5=90=8D=E5=B9=B6=E8=AE=B0?=
 =?UTF-8?q?=E6=88=90=E5=8A=9F=EF=BC=8C=E5=8F=AA=E6=9C=89=E4=BA=A7=E7=89=A9?=
 =?UTF-8?q?=E7=BC=BA=E5=A4=B1/=E4=B8=8D=E5=85=A8=E6=89=8D=E7=AE=97?=
 =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E3=80=82=E5=8F=A6=EF=BC=9A=E7=9B=91=E6=8E=A7?=
 =?UTF-8?q?=E9=A1=B5=201=20=E7=A7=92=E8=BD=AE=E8=AF=A2=EF=BC=88=E5=B9=B6?=
 =?UTF-8?q?=E8=A1=8C=E5=8F=96=20status/job/runs=20+=20nvidia-smi=20?=
 =?UTF-8?q?=E7=BC=93=E5=AD=98=EF=BC=89=EF=BC=8CGPU=20=E6=95=B0=E5=AD=97=20?=
 =?UTF-8?q?2=20=E7=A7=92=E7=BC=93=E5=AD=98=E3=80=82?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 ml/gen/run_parallel.sh | 14 +++++++++++++-
 ml/monitor.html        | 14 ++++++++------
 ml/monitor.py          | 10 ++++++++--
 3 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/ml/gen/run_parallel.sh b/ml/gen/run_parallel.sh
index cedcc47..59cb8a7 100755
--- a/ml/gen/run_parallel.sh
+++ b/ml/gen/run_parallel.sh
@@ -29,9 +29,21 @@ while :; do
     echo "[gen 节拍] $(( (tl-t0)/60 )) 分钟  运行中 $alive/${#pids[@]} 进程  本批已写 $n 条  $last"
   fi
 done
+# 判定按产物而不是 wait 的退出码：2026-10-18 实测 28 个种子跑了 5 小时后全部 wait 报失败，
+# 但每个 .part 都是完整的 1786 行（node 打了 done、mv 却没执行）——Git Bash 长跑后 wait 的返回码不可靠。
 fail=0
 for k in "${!pids[@]}"; do
-  if ! wait ${pids[$k]}; then echo "[gen] 种子 ${seeds[$k]} 失败，最后几行："; tail -n 5 ml/data/log_${seeds[$k]}.txt || true; fail=1; fi
+  s=${seeds[$k]}; f=ml/data/shard_$(printf %04d $s).jsonl
+  wait ${pids[$k]} || true
+  if [ -f "$f" ]; then continue; fi
+  if [ -f "$f.part" ]; then
+    n=$(wc -l < "$f.part")
+    if [ "$n" -ge "$N" ]; then mv "$f.part" "$f" && echo "[gen] 种子 $s 产物完整（$n 行），补改名" && continue; fi
+    echo "[gen] 种子 $s 失败：.part 只有 $n/$N 行，最后几行："; tail -n 5 ml/data/log_$s.txt || true
+  else
+    echo "[gen] 种子 $s 失败：没有产物，最后几行："; tail -n 5 ml/data/log_$s.txt || true
+  fi
+  fail=1
 done
 [ $fail = 0 ] || exit 6
 node ml/gen/stats.cjs ml/data/shard_*.jsonl > ml/data/stats.json
diff --git a/ml/monitor.html b/ml/monitor.html
index d078abf..3beb255 100644
--- a/ml/monitor.html
+++ b/ml/monitor.html
@@ -161,7 +161,7 @@
   <span class="sp"></span>
   <label class="ctl">平滑 <input id="smooth" type="range" min="0" max="0.99" step="0.01" value="0.85"><span id="smoothv" style="font-family:var(--mono);width:30px">0.85</span></label>
   <button id="logy" title="损失图纵轴取对数">对数轴</button>
-  <button id="auto" class="on" title="每 5 秒刷新"><span>●</span>自动刷新</button>
+  <button id="auto" class="on" title="每 1 秒刷新"><span>●</span>自动刷新</button>
   <button id="btnStart" class="primary" title="一键运行流水线">▶ 开始训练</button>
   <button id="btnStop" class="danger" style="display:none" title="停止正在运行的流水线">■ 停止</button>
 </header>
@@ -464,10 +464,12 @@ async function loadRuns() {
   if (!r) { cur = null; $('charts').innerHTML = ''; $('kpis').innerHTML = ''; $('prog').innerHTML = '<h2><span>当前运行 · 训练曲线</span></h2>' + empty('还没有训练日志：预训练开始后这里出现损失、吞吐和评估曲线。'); }
 }
 
+let polling = false;
 async function poll() {
+  if (polling) return; polling = true;      // 1 秒一轮：上一轮没回来就跳过，避免请求堆叠
   try {
-    lastStatus = await api('/api/status'); applyStageMeta(lastStatus.stage_meta);
-    job = await api('/api/job').catch(() => null);
+    const [st, jb] = await Promise.all([api('/api/status'), api('/api/job').catch(() => null)]);
+    lastStatus = st; applyStageMeta(st.stage_meta); job = jb;
     await loadRuns();
     if (cur) {
       const d = await api(`/api/log?run=${encodeURIComponent(cur.name)}&offset=${offset}`);
@@ -487,7 +489,7 @@ async function poll() {
     await pollConsole();
   } catch (err) {
     setPill('dead', '连不上控制台（双击 ml/console.bat 或 python -m ml.monitor）'); console.error(err);
-  }
+  } finally { polling = false; }
 }
 function setPill(cls, text) { const st = $('state'); st.className = 'pill ' + cls; st.lastElementChild.textContent = text; }
 
@@ -674,8 +676,8 @@ $('smooth').oninput = () => { $('smoothv').textContent = smoothv().toFixed(2); r
 $('logy').onclick = () => { logyOn = !logyOn; $('logy').classList.toggle('on', logyOn); render(); };
 $('run').onchange = () => { userPicked = true; cur = null; poll(); };
 $('scenario').onchange = () => { api.scenario($('scenario').value); cur = null; userPicked = false; conOff = 0; $('run').innerHTML = ''; poll(); };
-$('auto').onclick = () => { const on = !$('auto').classList.contains('on'); $('auto').classList.toggle('on', on); clearInterval(timer); if (on) timer = setInterval(poll, 5000); };
-timer = setInterval(poll, 5000); poll();
+$('auto').onclick = () => { const on = !$('auto').classList.contains('on'); $('auto').classList.toggle('on', on); clearInterval(timer); if (on) timer = setInterval(poll, 1000); };
+timer = setInterval(poll, 1000); poll();
 
 /* =====================================================================
    示例数据：两种场景，每次刷新让"运行中"的那条日志向前推进。
diff --git a/ml/monitor.py b/ml/monitor.py
index f035bcf..f03ee89 100644
--- a/ml/monitor.py
+++ b/ml/monitor.py
@@ -68,7 +68,12 @@ def read_log(name, offset):
     return {'records': recs, 'offset': offset + end, 'size': size, 'reset': offset == 0, 'mtime': os.path.getmtime(f)}
 
 
+_GPU_CACHE = {'t': 0.0, 'v': []}
+
+
 def gpu():
+    # 网页 1 秒轮询：nvidia-smi 每次要 150–600 ms，缓存 2 秒（GPU 数字不需要更快），避免每轮都起子进程
+    if time.time() - _GPU_CACHE['t'] < 2.0: return _GPU_CACHE['v']
     try:
         q = 'name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw'
         r = subprocess.run(['nvidia-smi', f'--query-gpu={q}', '--format=csv,noheader,nounits'], capture_output=True, text=True, timeout=3)
@@ -77,9 +82,10 @@ def gpu():
             n, u, mu, mt, t, p = [x.strip() for x in line.split(',')]
             num = lambda x: float(x) if re.fullmatch(r'[\d.]+', x) else None
             out.append({'name': n, 'util': num(u), 'mem_used': num(mu), 'mem_total': num(mt), 'temp': num(t), 'power': num(p)})
-        return out
     except Exception:
-        return []
+        out = []
+    _GPU_CACHE.update(t=time.time(), v=out)
+    return out
 
 
 def last_rec(name):