feat(train/monitor): 评估峰值不再撑爆整卡 + 从指定阶段重训 + 闭环误差诊断
- train.py:evaluate 改 bs=64,结束后 empty_cache()。WDDM 下 15.9 GiB 的评估峰值用完不退, 整卡顶到 96% 后驱动把训练张量换出(实测 Local 18.8 GB / Shared 1.9 GB),吞吐从 5.5k 掉到 444 samples/s、8000 步 eval 卡死 68 分钟;改后峰值 5.32 GiB、eval 106 s、吞吐 6.4–7.0k。 - pstate/pipeline:新增 --fresh-from=<阶段>:只把该阶段及之后的产物移进 ml/archive/(不删), 与参数失效取并集;配合 FROM 实现"只重训某一阶段,之后全部重做"(沙盒实测选 train 只归档 ml/runs/base)。 - monitor:运行方式加「从指定阶段重训」+ 起始阶段下拉;TARGET_SAMPLES 默认改 200 万; 「最好 X%」改标最好那次 eval 的步数(原来错标当前步,18,000 步显示成 0.2% 的出处)。 连同工作区已有的停止补扫 sweep_nt(按命令行补杀 git-bash 下逃逸的孙进程)一并提交。 - eval/diag_tail.py:闭环末位误差 / 是否进过 5 mm 窗口的逐题诊断(手动运行,不进流水线)。
This commit is contained in:
+31
-8
@@ -363,7 +363,7 @@ def status():
|
||||
STAGES = pstate.ORDER
|
||||
ENV_KEYS = {'N_TASKS': r'\d+', 'NPROC': r'\d+', 'TARGET_SAMPLES': r'\d+', 'CFG': r'ml/configs/(base|wta|small)\.json',
|
||||
'CONSENSUS': r'[01]?', 'ROUNDS': r'\d+', 'APG_STEPS': r'\d+', 'MINE_N': r'\d+', 'MIN_GAP_MM': r'\d+(\.\d+)?',
|
||||
'FRESH': r'[01]', 'R2_STRICT': r'[01]', 'AUTO_WTA': r'[01]'}
|
||||
'FRESH': r'[01]', 'FRESH_FROM': r'(setup|kin|r4|gate1|gen|prep|train|m5|apg|mine|export|bench|accept)', 'R2_STRICT': r'[01]', 'AUTO_WTA': r'[01]'}
|
||||
CONSOLE = os.path.join(ROOT, RUNS, 'console.log')
|
||||
|
||||
|
||||
@@ -419,6 +419,29 @@ def lock_info():
|
||||
return dict(lk, alive=pstate.lock_alive(lk), age_s=time.time() - lk.get('beat', 0))
|
||||
|
||||
|
||||
NT_NAMES = "'node.exe','xargs.exe','bash.exe','sh.exe','timeout.exe'"
|
||||
NT_SWEEP = (r"pipeline\.sh|run_prep\.sh|run_parallel\.sh|build_dataset\.cjs|"
|
||||
r"gen_dataset\.cjs|teacher\.cjs|policy_bench\.cjs")
|
||||
|
||||
|
||||
def sweep_nt(keep_pid):
|
||||
"""taskkill /T 在 Git Bash 下只杀到 bash,xargs/node 孙进程会逃(10-18 实测:停止后还有 28 个
|
||||
gen_dataset 在跑;当晚二次踩到:两批 build_dataset 抢同一批 shard)。按命令行补扫整条流水线,
|
||||
keep_pid 的祖先链(= 控制台自己拉起的 monitor bash)不动。先杀包装再杀 node,否则 node 一死
|
||||
xargs 会立刻补下一个 shard。返回仍存活个数,-1 = pwsh 没跑成。"""
|
||||
q = ("$anc=@(); $c=%d; while ($c -gt 0 -and ($p=Get-CimInstance Win32_Process -Filter \"ProcessId=$c\" "
|
||||
"-ErrorAction SilentlyContinue)) { $anc+=$p.ProcessId; $c=$p.ParentProcessId }; "
|
||||
"$m={ $_.Name -in %s -and $_.CommandLine -match '%s' -and $anc -notcontains $_.ProcessId }; "
|
||||
"foreach ($x in @(Get-CimInstance Win32_Process | Where-Object $m | "
|
||||
"Sort-Object { if ($_.Name -eq 'node.exe') { 1 } else { 0 } })) "
|
||||
"{ Stop-Process -Id $x.ProcessId -Force -ErrorAction SilentlyContinue }; "
|
||||
"Start-Sleep -Seconds 2; "
|
||||
"@(Get-CimInstance Win32_Process | Where-Object $m).Count" % (int(keep_pid), NT_NAMES, NT_SWEEP))
|
||||
r = subprocess.run(['pwsh', '-NoProfile', '-Command', q], capture_output=True, text=True, timeout=90)
|
||||
lines = [s for s in (r.stdout or '').split() if s.isdigit()]
|
||||
return int(lines[-1]) if lines else -1
|
||||
|
||||
|
||||
def kill_session(sid):
|
||||
"""POSIX:结束会话 sid 里的所有进程(含 timeout 另起的进程组)。"""
|
||||
pids = []
|
||||
@@ -496,14 +519,14 @@ class Job:
|
||||
with self.lock:
|
||||
if os.name == 'nt':
|
||||
subprocess.run(['taskkill', '/T', '/F', '/PID', str(pid)], capture_output=True)
|
||||
# Git Bash 下 taskkill /T 只杀到 bash,node 孙进程会逃掉(2026-10-18 实测:点了停止还有 28 个
|
||||
# gen_dataset 在跑)。按命令行补扫:只动本仓库 ml/ 下的脚本,绝不碰用户自己的 node 进程。
|
||||
# 补扫漏网的子孙;有残留就写进网页日志,别让用户以为停干净了
|
||||
try:
|
||||
q = ("Get-CimInstance Win32_Process -Filter \"Name='node.exe'\" | "
|
||||
"Where-Object { $_.CommandLine -match 'gen_dataset\\.cjs|run_parallel\\.sh|teacher\\.cjs|policy_bench\\.cjs' } | "
|
||||
"ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }")
|
||||
subprocess.run(['pwsh', '-NoProfile', '-Command', q], capture_output=True, timeout=60)
|
||||
except Exception: pass
|
||||
left = sweep_nt(os.getpid())
|
||||
except Exception:
|
||||
left = -1
|
||||
if left != 0:
|
||||
with open(CONSOLE, 'a', encoding='utf-8') as f:
|
||||
f.write(f'\n[停止补扫:仍有 {left} 个流水线子进程存活(-1=补扫没跑成),把这条发我]\n')
|
||||
else:
|
||||
if ours:
|
||||
kill_session(pid)
|
||||
|
||||
Reference in New Issue
Block a user