fix: gen 判定改按产物而不是 wait 退出码——实测 28 个种子跑 5 小时后全部 wait 报失败,但每个 .part 都是完整的 1786 行(node 打了 done、mv 没执行),Git Bash 长跑后 wait 返回码不可靠;现在只要 .part 行数达标就补改名并记成功,只有产物缺失/不全才算失败。另:监控页 1 秒轮询(并行取 status/job/runs + nvidia-smi 缓存),GPU 数字 2 秒缓存。

This commit is contained in:
cbb
2026-10-18 13:48:30 +08:00
parent b25a7a8bfa
commit c128288c1b
3 changed files with 29 additions and 9 deletions
+8 -2
View File
@@ -68,7 +68,12 @@ def read_log(name, offset):
return {'records': recs, 'offset': offset + end, 'size': size, 'reset': offset == 0, 'mtime': os.path.getmtime(f)}
_GPU_CACHE = {'t': 0.0, 'v': []}
def gpu():
# 网页 1 秒轮询:nvidia-smi 每次要 150–600 ms,缓存 2 秒(GPU 数字不需要更快),避免每轮都起子进程
if time.time() - _GPU_CACHE['t'] < 2.0: return _GPU_CACHE['v']
try:
q = 'name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw'
r = subprocess.run(['nvidia-smi', f'--query-gpu={q}', '--format=csv,noheader,nounits'], capture_output=True, text=True, timeout=3)
@@ -77,9 +82,10 @@ def gpu():
n, u, mu, mt, t, p = [x.strip() for x in line.split(',')]
num = lambda x: float(x) if re.fullmatch(r'[\d.]+', x) else None
out.append({'name': n, 'util': num(u), 'mem_used': num(mu), 'mem_total': num(mt), 'temp': num(t), 'power': num(p)})
return out
except Exception:
return []
out = []
_GPU_CACHE.update(t=time.time(), v=out)
return out
def last_rec(name):