Files
Robot_Arm/ml/monitor.py
T

316 lines
18 KiB
Python
Raw Normal View History

"""训练控制台(本地网页,零第三方依赖,离线可用):一键开始全流程、看进度、可停止。
双击 ml/console.bat(Windows)或运行 bash ml/console.sh(Linux/WSL)→ 自动打开 http://localhost:8100
也可:python -m ml.monitor [--port 8100] [--host 0.0.0.0] [--open]
网页上点「开始训练」→ 选起止阶段与参数 → 后台执行 bash ml/pipeline.sh all(FROM/TO 环境变量),
输出写 ml/runs/console.log,网页实时显示;「停止」结束整个进程树。网页关掉不影响训练,重新打开即可继续看。
只读数据来源:
ml/runs/pipeline.json 各阶段开始/结束/返回码(pipeline.sh 自动写)→ 全流程进度
ml/runs/*/log.jsonl 预训练(base / wta / bench_*)与 APG(apg / apg_r*)逐步日志与闭环评估
ml/runs/m5_gate_*.json、mine/r*/report.json + gate.json、policy_bench.json、acceptance.json
ml/data/stats.json / consistency.json / r2.json / samples/index.json、nvidia-smi
安全:默认只监听 127.0.0.1;--host 0.0.0.0 时其他机器只能看,开始/停止只接受本机请求(--remote-control 放开)。
2026-10-18 新增(用户要求:训练全程可视化,并在网页上点按钮一键开始)。"""
import argparse, glob, json, os, re, shutil, signal, subprocess, sys, threading, time, webbrowser
from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse, parse_qs
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
HTML = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'monitor.html')
RUNS, DATA = 'ml/runs', 'ml/data'
def jload(p):
try:
with open(p, encoding='utf-8') as f: return json.load(f)
except Exception: return None
def kind_of(name):
if name.startswith('apg'): return 'apg'
if name.startswith('bench'): return 'bench'
return 'train'
def list_runs():
out = []
for f in glob.glob(os.path.join(ROOT, RUNS, '*', 'log.jsonl')):
d = os.path.basename(os.path.dirname(f)); st = os.stat(f)
out.append({'name': d, 'kind': kind_of(d), 'size': st.st_size, 'mtime': st.st_mtime,
'has_best': os.path.exists(os.path.join(os.path.dirname(f), 'best.pt')),
'diverged': os.path.exists(os.path.join(os.path.dirname(f), 'diverged.pt'))})
out.sort(key=lambda r: -r['mtime'])
return out
def read_log(name, offset):
if not re.fullmatch(r'[\w.\-]+', name or ''): return {'error': 'bad run'}
f = os.path.join(ROOT, RUNS, name, 'log.jsonl')
if not os.path.exists(f): return {'error': 'no log'}
size = os.path.getsize(f)
if offset > size: offset = 0 # 日志被新训练覆盖 → 从头读
recs = []
with open(f, 'rb') as fh:
fh.seek(offset); buf = fh.read(min(size - offset, 64 << 20))
end = buf.rfind(b'\n') + 1 # 只取完整行,半行留到下次
for line in buf[:end].decode('utf-8', 'replace').splitlines():
if line.strip():
try: recs.append(json.loads(line))
except Exception: pass
return {'records': recs, 'offset': offset + end, 'size': size, 'reset': offset == 0, 'mtime': os.path.getmtime(f)}
def gpu():
try:
q = 'name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw'
r = subprocess.run(['nvidia-smi', f'--query-gpu={q}', '--format=csv,noheader,nounits'], capture_output=True, text=True, timeout=3)
out = []
for line in r.stdout.strip().splitlines():
n, u, mu, mt, t, p = [x.strip() for x in line.split(',')]
num = lambda x: float(x) if re.fullmatch(r'[\d.]+', x) else None
out.append({'name': n, 'util': num(u), 'mem_used': num(mu), 'mem_total': num(mt), 'temp': num(t), 'power': num(p)})
return out
except Exception:
return []
def last_rec(name):
"""读日志最后一条非评估记录(只读文件尾 16 KB)。"""
f = os.path.join(ROOT, RUNS, name, 'log.jsonl')
try:
with open(f, 'rb') as fh:
fh.seek(0, 2); n = fh.tell(); fh.seek(max(0, n - 16384)); tail = fh.read().decode('utf-8', 'replace').splitlines()
for line in reversed(tail):
try:
r = json.loads(line)
if 'eval' not in r: return r
except Exception: pass
except Exception: pass
return None
def newer(pattern, t0):
return [f for f in glob.glob(os.path.join(ROOT, pattern)) if os.path.getmtime(f) >= t0 - 1]
def stage_progress(st, s):
"""当前阶段的进度:返回 {'frac': 0..1 或 None(无法计量), 'label': 文字}。"""
t0, info = s['start'], s.get('info') or {}
if st == 'gen':
n = 0
for f in newer(f'{DATA}/shard_*.jsonl', t0):
with open(f, 'rb') as fh: n += sum(1 for _ in fh)
tgt = info.get('target_tasks') or 0
return {'frac': min(1, n / tgt) if tgt else None, 'label': f'已生成 {n:,} / {tgt:,} 条任务'}
if st == 'prep':
nsh = len(glob.glob(os.path.join(ROOT, DATA, 'shard_*.jsonl')))
done = len(newer(f'{DATA}/samples/*.X.npy', t0))
if os.path.exists(os.path.join(ROOT, DATA, 'r2.json')) and newer(f'{DATA}/r2.json', t0): return {'frac': 1, 'label': '打包完成,R2 检查完成'}
return {'frac': min(0.95, done / nsh) if nsh else None, 'label': f'已打包 {done} / {nsh} 个分片' + (',R2 检查中' if nsh and done >= nsh else '')}
if st == 'train':
tag, bs = info.get('tag', 'base'), info.get('bench_steps', 2000)
b, m = last_rec(f'bench_{tag}') if newer(f'{RUNS}/bench_{tag}/log.jsonl', t0) else None, last_rec(tag) if newer(f'{RUNS}/{tag}/log.jsonl', t0) else None
if m and m.get('total'):
return {'frac': (bs + m['step']) / (bs + m['total']), 'label': f'正式训练 {m["step"]:,} / {m["total"]:,} 步', 'run': tag}
if b: return {'frac': b['step'] / (bs + 60000), 'label': f'测速 {b["step"]:,} / {bs:,} 步', 'run': f'bench_{tag}'}
return {'frac': 0, 'label': '加载数据中'}
if st == 'apg':
r = last_rec('apg') if newer(f'{RUNS}/apg/log.jsonl', t0) else None
return {'frac': r['it'] / r['total'] if r and r.get('total') else 0, 'label': f'APG {r["it"]:,} / {r["total"]:,} 步' if r else '准备中', 'run': 'apg'}
if st == 'mine':
R = info.get('rounds', 6)
k = len(newer(f'{RUNS}/mine/r*/gate.json', t0))
cur = sorted(newer(f'{RUNS}/apg_r*/log.jsonl', t0), key=os.path.getmtime)
frac_in, lab = 0, f'第 {k + 1} 轮:挖错出题'
if cur and len(cur) > k:
name = os.path.basename(os.path.dirname(cur[-1])); r = last_rec(name)
if r and r.get('total'): frac_in = 0.3 + 0.6 * r['it'] / r['total']; lab = f'第 {k + 1} 轮:APG {r["it"]:,} / {r["total"]:,} 步'
return {'frac': min(1, (k + frac_in) / R), 'label': f'{lab}(共最多 {R} 轮,提升不足会提前停)', 'run': name}
return {'frac': min(1, k / R), 'label': f'{lab}(共最多 {R} 轮,提升不足会提前停)'}
if st == 'm5':
tag = info.get('tag', 'base')
n = len(newer(f'{RUNS}/m5_{tag}_policy.json', t0)) + len(newer(f'{RUNS}/m5_{tag}_safety.json', t0))
return {'frac': n / 2, 'label': ['纯策略评估中', '加安全层评估中', '汇总'][min(n, 2)]}
return {'frac': None, 'label': '进行中'}
def pipeline():
d = jload(os.path.join(ROOT, RUNS, 'pipeline.json'))
if not d: return None
cur = d.get('current')
if cur and cur in d['stages']:
try: d['progress'] = stage_progress(cur, d['stages'][cur])
except Exception as e: d['progress'] = {'frac': None, 'label': f'进度读取失败:{e}'}
return d
def status():
P = lambda *a: os.path.join(ROOT, *a)
mine = []
for d in sorted(glob.glob(P(RUNS, 'mine', 'r*')), key=lambda x: int(re.sub(r'\D', '', os.path.basename(x)) or 0)):
rep, gate = jload(os.path.join(d, 'report.json')), jload(os.path.join(d, 'gate.json'))
if rep or gate:
mine.append({'round': os.path.basename(d), 'report': rep and {k: rep.get(k) for k in ('n', 'E1', 'E2', 'E3', 'E4', 'E5', 'ok', 'pool')},
'gate': gate and {k: gate.get(k) for k in ('gain_pp', 'A_drop_pp', 'fixed_kept', 'R7_left_share_shift_pp', 'R7_travel_median_ratio', 'accept')}})
stats = jload(P(DATA, 'stats.json'))
if stats: stats = {k: stats.get(k) for k in ('tasks', 'keep', 'gapDrop', 'minGapFilterMm', 'keepRate', 'samplesEvery3', 'samplesPerTask', 'projectedSamplesFor50k', 'consensusDrop', 'vertexMinGapMm', 'execS', 'travelDeg')}
shards = glob.glob(P(DATA, 'shard_*.jsonl'))
gen_lines = 0
for s in shards:
try:
with open(s, 'rb') as fh: gen_lines += sum(1 for _ in fh)
except Exception: pass
pb = jload(P(RUNS, 'policy_bench.json'))
if pb: pb = {k: {kk: v.get(kk) for kk in ('n', 'ok', 'reached', 'collisionFrames', 'brakeRate', 'recoveryRate', 'livelock', 'inferMsP99', 'safetyMsP99')}
for k, v in pb.items() if isinstance(v, dict) and 'n' in v}
return {'now': time.time(), 'gpu': gpu(), 'pipeline': pipeline(),
'data': {'consistency': (jload(P(DATA, 'consistency.json')) or {}).get('consistentByGateRule'),
'gen_tasks_written': gen_lines, 'shards': len(shards), 'stats': stats,
'index': jload(P(DATA, 'samples', 'index.json')), 'r2': jload(P(DATA, 'r2.json'))},
'm5': [x for x in (jload(f) for f in sorted(glob.glob(P(RUNS, 'm5_gate_*.json')))) if x],
'selected': open(P(RUNS, 'selected_ckpt')).read().strip() if os.path.exists(P(RUNS, 'selected_ckpt')) else None,
'mine': mine, 'bench': pb, 'acceptance': jload(P(RUNS, 'acceptance.json'))}
STAGES = ['setup', 'kin', 'gate1', 'gen', 'prep', 'train', 'm5', 'apg', 'mine', 'export', 'bench', 'accept']
ENV_KEYS = {'N_TASKS': r'\d+', 'NPROC': r'\d+', 'TARGET_SAMPLES': r'\d+', 'CFG': r'ml/configs/(base|wta)\.json',
'CONSENSUS': r'[01]?', 'ROUNDS': r'\d+', 'APG_STEPS': r'\d+', 'MINE_N': r'\d+'}
CONSOLE = os.path.join(ROOT, RUNS, 'console.log')
def find_bash():
"""Windows 优先 Git Bash(System32\\bash.exe 是 WSL,不能用);其他系统用 PATH 里的 bash。"""
if os.name == 'nt':
for p in [os.environ.get('BASH', ''), r'C:\Program Files\Git\bin\bash.exe', r'C:\Program Files (x86)\Git\bin\bash.exe',
os.path.expandvars(r'%LOCALAPPDATA%\Programs\Git\bin\bash.exe')]:
if p and os.path.exists(p): return p
g = shutil.which('git')
if g:
p = os.path.join(os.path.dirname(os.path.dirname(g)), 'bin', 'bash.exe')
if os.path.exists(p): return p
return None
return shutil.which('bash')
class Job:
def __init__(self):
self.p = None; self.info = {}; self.lock = threading.Lock()
def state(self):
with self.lock:
run = self.p is not None and self.p.poll() is None
if self.p is not None and not run and self.info.get('rc') is None:
self.info['rc'] = self.p.returncode; self.info['ended'] = time.time()
return {'running': run, **self.info, 'bash': find_bash() is not None}
def start(self, frm, to, env):
with self.lock:
if self.p is not None and self.p.poll() is None: return {'error': '已经在运行'}
pj = jload(os.path.join(ROOT, RUNS, 'pipeline.json')) or {}
cur = pj.get('current')
if cur and time.time() - (pj['stages'].get(cur, {}).get('start') or 0) < 7 * 86400 and not self.p:
# 终端里另起的流水线还在跑(或上次被强杀没写结束):看有没有最近更新的日志
fresh = [r for r in list_runs() if time.time() - r['mtime'] < 120]
if fresh: return {'error': f'检测到终端里正在运行阶段 {cur}(日志 {fresh[0]["name"]} 2 分钟内有更新),请先在终端结束它'}
if frm not in STAGES or to not in STAGES or STAGES.index(frm) > STAGES.index(to): return {'error': '起止阶段无效'}
bash = find_bash()
if not bash: return {'error': '找不到 bash:Windows 请安装 Git for Windows(自带 Git Bash)'}
e = dict(os.environ, FROM=frm, TO=to, MONITOR='0', PYTHONUNBUFFERED='1', PYTHONIOENCODING='utf-8')
if ' ' not in sys.executable: e['PYTHON'] = sys.executable.replace('\\', '/') # 与控制台同一个 Python
for k, v in (env or {}).items():
if k in ENV_KEYS and v not in (None, '') and re.fullmatch(ENV_KEYS[k], str(v)): e[k] = str(v)
os.makedirs(os.path.dirname(CONSOLE), exist_ok=True)
with open(CONSOLE, 'w', encoding='utf-8') as f: f.write(f'$ FROM={frm} TO={to} bash ml/pipeline.sh all [{time.strftime("%Y-%m-%d %H:%M:%S")}]\n')
log = open(CONSOLE, 'a', encoding='utf-8') # 追加模式:网页写入的"已停止"提示不会被子进程覆盖
kw = {'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP} if os.name == 'nt' else {'start_new_session': True}
self.p = subprocess.Popen([bash, 'ml/pipeline.sh', 'all'], cwd=ROOT, env=e, stdout=log, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, **kw)
self.info = {'pid': self.p.pid, 'from': frm, 'to': to, 'started': time.time(), 'rc': None, 'ended': None,
'env': {k: e[k] for k in ENV_KEYS if k in e}}
return {'ok': True, 'pid': self.p.pid}
def stop(self):
with self.lock:
if self.p is None or self.p.poll() is not None: return {'error': '没有在运行'}
PJ = os.path.join(ROOT, RUNS, 'pipeline.json')
cur = (jload(PJ) or {}).get('current') # 先记下正在跑的阶段:被杀的子进程可能以返回码 0 写结束
if os.name == 'nt':
subprocess.run(['taskkill', '/T', '/F', '/PID', str(self.p.pid)], capture_output=True)
else:
try: os.killpg(self.p.pid, signal.SIGTERM)
except ProcessLookupError: pass
try: self.p.wait(10)
except subprocess.TimeoutExpired: os.killpg(self.p.pid, signal.SIGKILL)
self.info['stopped'] = True
with open(CONSOLE, 'a', encoding='utf-8') as f: f.write(f'\n[已在网页上停止 {time.strftime("%H:%M:%S")}]\n')
time.sleep(0.5); pj = jload(PJ)
if pj and cur and cur in pj['stages']: # 该阶段记为"已停止"(返回码 -15)
pj['stages'][cur].update(end=time.time(), rc=-15); pj['current'] = None
with open(PJ, 'w', encoding='utf-8') as f: json.dump(pj, f, ensure_ascii=False, indent=1)
return {'ok': True}
JOB = Job()
def console(offset):
if not os.path.exists(CONSOLE): return {'text': '', 'offset': 0, 'reset': True}
size = os.path.getsize(CONSOLE); reset = offset > size or offset == 0
if reset: offset = max(0, size - 65536) # 首次只给最后 64 KB
with open(CONSOLE, 'rb') as f: f.seek(offset); b = f.read(min(size - offset, 1 << 20))
return {'text': b.decode('utf-8', 'replace').replace('\r', ''), 'offset': offset + len(b), 'reset': reset}
class H(BaseHTTPRequestHandler):
def log_message(self, *a): pass
def send(self, code, body, ctype):
b = body if isinstance(body, bytes) else body.encode('utf-8')
self.send_response(code); self.send_header('Content-Type', ctype); self.send_header('Content-Length', str(len(b)))
self.send_header('Cache-Control', 'no-store'); self.end_headers(); self.wfile.write(b)
def do_GET(self):
u = urlparse(self.path); q = parse_qs(u.query)
try:
if u.path in ('/', '/index.html'):
with open(HTML, 'rb') as f: return self.send(200, f.read(), 'text/html; charset=utf-8')
if u.path == '/api/runs': return self.send(200, json.dumps(list_runs()), 'application/json')
if u.path == '/api/log':
return self.send(200, json.dumps(read_log(q.get('run', [''])[0], int(q.get('offset', ['0'])[0]))), 'application/json')
if u.path == '/api/status': return self.send(200, json.dumps(status()), 'application/json')
if u.path == '/api/job': return self.send(200, json.dumps(JOB.state()), 'application/json')
if u.path == '/api/console': return self.send(200, json.dumps(console(int(q.get('offset', ['0'])[0]))), 'application/json')
self.send(404, 'not found', 'text/plain')
except Exception as e:
self.send(500, json.dumps({'error': str(e)}), 'application/json')
def do_POST(self):
u = urlparse(self.path)
if not (self.server.remote_control or self.client_address[0] in ('127.0.0.1', '::1')):
return self.send(403, json.dumps({'error': '开始/停止只接受本机操作'}), 'application/json')
try:
n = int(self.headers.get('Content-Length') or 0); body = json.loads(self.rfile.read(n) or b'{}')
if u.path == '/api/start': r = JOB.start(body.get('from', 'kin'), body.get('to', 'accept'), body.get('env') or {})
elif u.path == '/api/stop': r = JOB.stop()
else: return self.send(404, 'not found', 'text/plain')
self.send(200 if 'ok' in r else 409, json.dumps(r, ensure_ascii=False), 'application/json')
except Exception as e:
self.send(500, json.dumps({'error': str(e)}), 'application/json')
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--port', type=int, default=8100); ap.add_argument('--host', default='127.0.0.1')
ap.add_argument('--open', action='store_true', help='启动后自动打开浏览器')
ap.add_argument('--remote-control', action='store_true', help='允许其他机器点开始/停止(默认只能看)')
a = ap.parse_args()
srv = ThreadingHTTPServer((a.host, a.port), H); srv.remote_control = a.remote_control
url = f'http://localhost:{a.port}/'
print(f'训练控制台:{url} (关闭本窗口会停止控制台;已开始的训练会一并结束,先在网页上确认跑完)', flush=True)
if a.open: threading.Timer(0.8, lambda: webbrowser.open(url)).start()
try: srv.serve_forever()
except KeyboardInterrupt: pass
if __name__ == '__main__':
main()