feat(monitor): 用时改为按阶段累计——"总用时"=各阶段之和、"本步用时"=该阶段累计,停止后再"继续训练"不再归零(训练跑 5 万步停一次,之前那段的时间会保留);只有产物被归档、要从头重做的阶段才清零;本次改动前完成的老记录用 start/end 兜底;停止与流水线退出两处记账幂等(实测 180 s 不翻倍)

This commit is contained in:
cbb
2026-10-18 16:59:21 +08:00
parent 21ad209747
commit db0bbbc15d
3 changed files with 51 additions and 13 deletions
+18
View File
@@ -216,9 +216,27 @@ def stage_progress(st, s):
return {'frac': None, 'label': '进行中'}
def stage_elapsed(pl, now=None):
"""每个阶段的累计用时(秒):pipeline.json 的 time{阶段: 秒} 是不含当前段的累计,运行中的阶段
再把"这一段"实时加上。老记录(没有 time 字段)用 start/end 的差值兜底。
网页的「总用时」=各阶段之和,「本步用时」=当前阶段这条 —— 确定/续跑都不归零。"""
now = now or time.time()
acc = pl.get('time') or {}
cur = pl.get('current')
out = {}
for k, r in (pl.get('stages') or {}).items():
t = acc.get(k)
if t is None and r.get('start') and r.get('end'): t = r['end'] - r['start']
t = t or 0
if k == cur and r.get('start') and r.get('end') is None: t += max(0.0, now - r['start'])
out[k] = round(t, 1)
return out
def pipeline():
d = jload(os.path.join(ROOT, RUNS, 'pipeline.json'))
if not d: return None
d['elapsed'] = stage_elapsed(d)
cur = d.get('current')
if cur and cur in d['stages']:
try: d['progress'] = stage_progress(cur, d['stages'][cur])