fix: gen 判定改按产物而不是 wait 退出码——实测 28 个种子跑 5 小时后全部 wait 报失败,但每个 .part 都是完整的 1786 行(node 打了 done、mv 没执行),Git Bash 长跑后 wait 返回码不可靠;现在只要 .part 行数达标就补改名并记成功,只有产物缺失/不全才算失败。另:监控页 1 秒轮询(并行取 status/job/runs + nvidia-smi 缓存),GPU 数字 2 秒缓存。

This commit is contained in:
cbb
2026-10-18 13:48:30 +08:00
parent b25a7a8bfa
commit c128288c1b
3 changed files with 29 additions and 9 deletions
+8 -6
View File
@@ -161,7 +161,7 @@
<span class="sp"></span>
<label class="ctl">平滑 <input id="smooth" type="range" min="0" max="0.99" step="0.01" value="0.85"><span id="smoothv" style="font-family:var(--mono);width:30px">0.85</span></label>
<button id="logy" title="损失图纵轴取对数">对数轴</button>
<button id="auto" class="on" title="每 5 秒刷新"><span>●</span>自动刷新</button>
<button id="auto" class="on" title="每 1 秒刷新"><span>●</span>自动刷新</button>
<button id="btnStart" class="primary" title="一键运行流水线">▶ 开始训练</button>
<button id="btnStop" class="danger" style="display:none" title="停止正在运行的流水线">■ 停止</button>
</header>
@@ -464,10 +464,12 @@ async function loadRuns() {
if (!r) { cur = null; $('charts').innerHTML = ''; $('kpis').innerHTML = ''; $('prog').innerHTML = '<h2><span>当前运行 · 训练曲线</span></h2>' + empty('还没有训练日志:预训练开始后这里出现损失、吞吐和评估曲线。'); }
}
let polling = false;
async function poll() {
if (polling) return; polling = true; // 1 秒一轮:上一轮没回来就跳过,避免请求堆叠
try {
lastStatus = await api('/api/status'); applyStageMeta(lastStatus.stage_meta);
job = await api('/api/job').catch(() => null);
const [st, jb] = await Promise.all([api('/api/status'), api('/api/job').catch(() => null)]);
lastStatus = st; applyStageMeta(st.stage_meta); job = jb;
await loadRuns();
if (cur) {
const d = await api(`/api/log?run=${encodeURIComponent(cur.name)}&offset=${offset}`);
@@ -487,7 +489,7 @@ async function poll() {
await pollConsole();
} catch (err) {
setPill('dead', '连不上控制台(双击 ml/console.bat 或 python -m ml.monitor)'); console.error(err);
}
} finally { polling = false; }
}
function setPill(cls, text) { const st = $('state'); st.className = 'pill ' + cls; st.lastElementChild.textContent = text; }
@@ -674,8 +676,8 @@ $('smooth').oninput = () => { $('smoothv').textContent = smoothv().toFixed(2); r
$('logy').onclick = () => { logyOn = !logyOn; $('logy').classList.toggle('on', logyOn); render(); };
$('run').onchange = () => { userPicked = true; cur = null; poll(); };
$('scenario').onchange = () => { api.scenario($('scenario').value); cur = null; userPicked = false; conOff = 0; $('run').innerHTML = ''; poll(); };
$('auto').onclick = () => { const on = !$('auto').classList.contains('on'); $('auto').classList.toggle('on', on); clearInterval(timer); if (on) timer = setInterval(poll, 5000); };
timer = setInterval(poll, 5000); poll();
$('auto').onclick = () => { const on = !$('auto').classList.contains('on'); $('auto').classList.toggle('on', on); clearInterval(timer); if (on) timer = setInterval(poll, 1000); };
timer = setInterval(poll, 1000); poll();
/* =====================================================================
示例数据:两种场景,每次刷新让"运行中"的那条日志向前推进。