fix: gen 判定改按产物而不是 wait 退出码——实测 28 个种子跑 5 小时后全部 wait 报失败,但每个 .part 都是完整的 1786 行(node 打了 done、mv 没执行),Git Bash 长跑后 wait 返回码不可靠;现在只要 .part 行数达标就补改名并记成功,只有产物缺失/不全才算失败。另:监控页 1 秒轮询(并行取 status/job/runs + nvidia-smi 缓存),GPU 数字 2 秒缓存。
This commit is contained in:
+8
-6
@@ -161,7 +161,7 @@
|
||||
<span class="sp"></span>
|
||||
<label class="ctl">平滑 <input id="smooth" type="range" min="0" max="0.99" step="0.01" value="0.85"><span id="smoothv" style="font-family:var(--mono);width:30px">0.85</span></label>
|
||||
<button id="logy" title="损失图纵轴取对数">对数轴</button>
|
||||
<button id="auto" class="on" title="每 5 秒刷新"><span>●</span>自动刷新</button>
|
||||
<button id="auto" class="on" title="每 1 秒刷新"><span>●</span>自动刷新</button>
|
||||
<button id="btnStart" class="primary" title="一键运行流水线">▶ 开始训练</button>
|
||||
<button id="btnStop" class="danger" style="display:none" title="停止正在运行的流水线">■ 停止</button>
|
||||
</header>
|
||||
@@ -464,10 +464,12 @@ async function loadRuns() {
|
||||
if (!r) { cur = null; $('charts').innerHTML = ''; $('kpis').innerHTML = ''; $('prog').innerHTML = '<h2><span>当前运行 · 训练曲线</span></h2>' + empty('还没有训练日志:预训练开始后这里出现损失、吞吐和评估曲线。'); }
|
||||
}
|
||||
|
||||
let polling = false;
|
||||
async function poll() {
|
||||
if (polling) return; polling = true; // 1 秒一轮:上一轮没回来就跳过,避免请求堆叠
|
||||
try {
|
||||
lastStatus = await api('/api/status'); applyStageMeta(lastStatus.stage_meta);
|
||||
job = await api('/api/job').catch(() => null);
|
||||
const [st, jb] = await Promise.all([api('/api/status'), api('/api/job').catch(() => null)]);
|
||||
lastStatus = st; applyStageMeta(st.stage_meta); job = jb;
|
||||
await loadRuns();
|
||||
if (cur) {
|
||||
const d = await api(`/api/log?run=${encodeURIComponent(cur.name)}&offset=${offset}`);
|
||||
@@ -487,7 +489,7 @@ async function poll() {
|
||||
await pollConsole();
|
||||
} catch (err) {
|
||||
setPill('dead', '连不上控制台(双击 ml/console.bat 或 python -m ml.monitor)'); console.error(err);
|
||||
}
|
||||
} finally { polling = false; }
|
||||
}
|
||||
function setPill(cls, text) { const st = $('state'); st.className = 'pill ' + cls; st.lastElementChild.textContent = text; }
|
||||
|
||||
@@ -674,8 +676,8 @@ $('smooth').oninput = () => { $('smoothv').textContent = smoothv().toFixed(2); r
|
||||
$('logy').onclick = () => { logyOn = !logyOn; $('logy').classList.toggle('on', logyOn); render(); };
|
||||
$('run').onchange = () => { userPicked = true; cur = null; poll(); };
|
||||
$('scenario').onchange = () => { api.scenario($('scenario').value); cur = null; userPicked = false; conOff = 0; $('run').innerHTML = ''; poll(); };
|
||||
$('auto').onclick = () => { const on = !$('auto').classList.contains('on'); $('auto').classList.toggle('on', on); clearInterval(timer); if (on) timer = setInterval(poll, 5000); };
|
||||
timer = setInterval(poll, 5000); poll();
|
||||
$('auto').onclick = () => { const on = !$('auto').classList.contains('on'); $('auto').classList.toggle('on', on); clearInterval(timer); if (on) timer = setInterval(poll, 1000); };
|
||||
timer = setInterval(poll, 1000); poll();
|
||||
|
||||
/* =====================================================================
|
||||
示例数据:两种场景,每次刷新让"运行中"的那条日志向前推进。
|
||||
|
||||
Reference in New Issue
Block a user