feat(monitor): 用时改为按阶段累计——"总用时"=各阶段之和、"本步用时"=该阶段累计,停止后再"继续训练"不再归零(训练跑 5 万步停一次,之前那段的时间会保留);只有产物被归档、要从头重做的阶段才清零;本次改动前完成的老记录用 start/end 兜底;停止与流水线退出两处记账幂等(实测 180 s 不翻倍)
This commit is contained in:
@@ -216,9 +216,27 @@ def stage_progress(st, s):
|
||||
return {'frac': None, 'label': '进行中'}
|
||||
|
||||
|
||||
def stage_elapsed(pl, now=None):
|
||||
"""每个阶段的累计用时(秒):pipeline.json 的 time{阶段: 秒} 是不含当前段的累计,运行中的阶段
|
||||
再把"这一段"实时加上。老记录(没有 time 字段)用 start/end 的差值兜底。
|
||||
网页的「总用时」=各阶段之和,「本步用时」=当前阶段这条 —— 确定/续跑都不归零。"""
|
||||
now = now or time.time()
|
||||
acc = pl.get('time') or {}
|
||||
cur = pl.get('current')
|
||||
out = {}
|
||||
for k, r in (pl.get('stages') or {}).items():
|
||||
t = acc.get(k)
|
||||
if t is None and r.get('start') and r.get('end'): t = r['end'] - r['start']
|
||||
t = t or 0
|
||||
if k == cur and r.get('start') and r.get('end') is None: t += max(0.0, now - r['start'])
|
||||
out[k] = round(t, 1)
|
||||
return out
|
||||
|
||||
|
||||
def pipeline():
|
||||
d = jload(os.path.join(ROOT, RUNS, 'pipeline.json'))
|
||||
if not d: return None
|
||||
d['elapsed'] = stage_elapsed(d)
|
||||
cur = d.get('current')
|
||||
if cur and cur in d['stages']:
|
||||
try: d['progress'] = stage_progress(cur, d['stages'][cur])
|
||||
|
||||
Reference in New Issue
Block a user