2026-10-17 17:15:16 +00:00
"""训练控制台(本地网页,零第三方依赖,离线可用):一键开始全流程、看进度、可停止。
双击 ml/console.bat( Windows)或运行 bash ml/console.sh( Linux/WSL)→ 自动打开 http://localhost:8100
也可:python -m ml.monitor [--port 8100] [--host 0.0.0.0] [--open]
2026-10-17 18:57:48 +00:00
网页上点「开始训练」→ 选参数 → 后台执行 bash ml/pipeline.sh all(已完成的阶段自动跳过 = 续跑;勾「从头重跑」= FRESH=1),
输出写 ml/runs/console.log,网页实时显示;「停止」结束整个进程树。网页和控制台窗口关掉都不影响训练,重新打开即可继续看。
2026-10-17 17:15:16 +00:00
只读数据来源:
ml/runs/pipeline.json 各阶段开始/结束/返回码(pipeline.sh 自动写)→ 全流程进度
ml/runs/*/log.jsonl 预训练(base / wta / bench_*)与 APG( apg / apg_r*)逐步日志与闭环评估
ml/runs/m5_gate_*.json、mine/r*/report.json + gate.json、policy_bench.json、acceptance.json
ml/data/stats.json / consistency.json / r2.json / samples/index.json、nvidia-smi
安全:默认只监听 127.0.0.1; --host 0.0.0.0 时其他机器只能看,开始/停止只接受本机请求(--remote-control 放开)。
2026-10-17 18:57:48 +00:00
2026-10-18 新增(用户要求:训练全程可视化,并在网页上点按钮一键开始)。
2026-10-18 第五轮(审查第四轮):
N-7 是否在跑看 ml/runs/pipeline.lock(与终端同一把锁),不再靠"日志 120 s 内有更新"去猜;
N-8 开始/停止要求自定义头 X-Console + 同源 Origin/Sec-Fetch-Site,外站网页发不进来;
N-10 启动自检(训练用 Python 能否 import torch、GPU、node、bash),不过就拒绝「开始训练」并在顶栏标红;
N-11 Windows 子进程用 CREATE_NO_WINDOW 自带隐藏控制台:关掉控制台黑窗口训练照跑,重开控制台可接着看、可停止;
N-9 生成阶段的行数按文件增量统计(只读新增字节),其他阶段不扫 shard;
N-12 停止后的状态改写走 ml.pstate(单一写者、原子替换);N-14 阶段名/估时/权重由 ml.pstate.STAGES 下发。"""
2026-10-17 17:15:16 +00:00
import argparse , glob , json , os , re , shutil , signal , subprocess , sys , threading , time , webbrowser
2026-10-17 18:57:48 +00:00
sys . path . insert ( 0 , os . path . dirname ( os . path . dirname ( os . path . abspath ( __file__ ))))
from ml import pstate
2026-10-17 17:15:16 +00:00
from http.server import ThreadingHTTPServer , BaseHTTPRequestHandler
from urllib.parse import urlparse , parse_qs
ROOT = os . path . dirname ( os . path . dirname ( os . path . abspath ( __file__ )))
HTML = os . path . join ( os . path . dirname ( os . path . abspath ( __file__ )), 'monitor.html' )
RUNS , DATA = 'ml/runs' , 'ml/data'
def jload ( p ):
try :
with open ( p , encoding = 'utf-8' ) as f : return json . load ( f )
except Exception : return None
def kind_of ( name ):
if name . startswith ( 'apg' ): return 'apg'
if name . startswith ( 'bench' ): return 'bench'
return 'train'
def list_runs ():
out = []
for f in glob . glob ( os . path . join ( ROOT , RUNS , '*' , 'log.jsonl' )):
d = os . path . basename ( os . path . dirname ( f )); st = os . stat ( f )
out . append ({ 'name' : d , 'kind' : kind_of ( d ), 'size' : st . st_size , 'mtime' : st . st_mtime ,
'has_best' : os . path . exists ( os . path . join ( os . path . dirname ( f ), 'best.pt' )),
'diverged' : os . path . exists ( os . path . join ( os . path . dirname ( f ), 'diverged.pt' ))})
out . sort ( key = lambda r : - r [ 'mtime' ])
return out
def read_log ( name , offset ):
if not re . fullmatch ( r '[\w.\-]+' , name or '' ): return { 'error' : 'bad run' }
f = os . path . join ( ROOT , RUNS , name , 'log.jsonl' )
if not os . path . exists ( f ): return { 'error' : 'no log' }
size = os . path . getsize ( f )
if offset > size : offset = 0 # 日志被新训练覆盖 → 从头读
recs = []
with open ( f , 'rb' ) as fh :
fh . seek ( offset ); buf = fh . read ( min ( size - offset , 64 << 20 ))
end = buf . rfind ( b ' \n ' ) + 1 # 只取完整行,半行留到下次
for line in buf [: end ] . decode ( 'utf-8' , 'replace' ) . splitlines ():
if line . strip ():
try : recs . append ( json . loads ( line ))
except Exception : pass
return { 'records' : recs , 'offset' : offset + end , 'size' : size , 'reset' : offset == 0 , 'mtime' : os . path . getmtime ( f )}
def gpu ():
try :
q = 'name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw'
r = subprocess . run ([ 'nvidia-smi' , f '--query-gpu= { q } ' , '--format=csv,noheader,nounits' ], capture_output = True , text = True , timeout = 3 )
out = []
for line in r . stdout . strip () . splitlines ():
n , u , mu , mt , t , p = [ x . strip () for x in line . split ( ',' )]
num = lambda x : float ( x ) if re . fullmatch ( r '[\d.]+' , x ) else None
out . append ({ 'name' : n , 'util' : num ( u ), 'mem_used' : num ( mu ), 'mem_total' : num ( mt ), 'temp' : num ( t ), 'power' : num ( p )})
return out
except Exception :
return []
def last_rec ( name ):
"""读日志最后一条非评估记录(只读文件尾 16 KB)。"""
f = os . path . join ( ROOT , RUNS , name , 'log.jsonl' )
try :
with open ( f , 'rb' ) as fh :
fh . seek ( 0 , 2 ); n = fh . tell (); fh . seek ( max ( 0 , n - 16384 )); tail = fh . read () . decode ( 'utf-8' , 'replace' ) . splitlines ()
for line in reversed ( tail ):
try :
r = json . loads ( line )
if 'eval' not in r : return r
except Exception : pass
except Exception : pass
return None
2026-10-17 18:57:48 +00:00
class LineCount :
"""按文件增量数行:记住每个文件已读到的偏移与行数,只读新增字节(审查 N-9:此前每 5 s 全量重读全部 shard)。"""
def __init__ ( self ): self . c = {}; self . lock = threading . Lock ()
def count ( self , f ):
with self . lock :
try : size = os . path . getsize ( f )
except OSError : return 0
off , n = self . c . get ( f , ( 0 , 0 ))
if size < off : off , n = 0 , 0
if size > off :
with open ( f , 'rb' ) as fh :
fh . seek ( off )
while True :
b = fh . read ( 1 << 22 )
if not b : break
n += b . count ( b ' \n ' ); off += len ( b )
self . c [ f ] = ( off , n )
return n
LC = LineCount ()
2026-10-17 17:15:16 +00:00
def newer ( pattern , t0 ):
return [ f for f in glob . glob ( os . path . join ( ROOT , pattern )) if os . path . getmtime ( f ) >= t0 - 1 ]
def stage_progress ( st , s ):
"""当前阶段的进度:返回 {'frac': 0..1 或 None(无法计量), 'label': 文字}。"""
t0 , info = s [ 'start' ], s . get ( 'info' ) or {}
if st == 'gen' :
2026-10-17 18:57:48 +00:00
n = sum ( LC . count ( f ) for f in newer ( f ' { DATA } /shard_*.jsonl' , t0 ) + newer ( f ' { DATA } /shard_*.jsonl.part' , t0 ))
2026-10-17 17:15:16 +00:00
tgt = info . get ( 'target_tasks' ) or 0
return { 'frac' : min ( 1 , n / tgt ) if tgt else None , 'label' : f '已生成 { n : , } / { tgt : , } 条任务' }
if st == 'prep' :
nsh = len ( glob . glob ( os . path . join ( ROOT , DATA , 'shard_*.jsonl' )))
done = len ( newer ( f ' { DATA } /samples/*.X.npy' , t0 ))
if os . path . exists ( os . path . join ( ROOT , DATA , 'r2.json' )) and newer ( f ' { DATA } /r2.json' , t0 ): return { 'frac' : 1 , 'label' : '打包完成,R2 检查完成' }
return { 'frac' : min ( 0.95 , done / nsh ) if nsh else None , 'label' : f '已打包 { done } / { nsh } 个分片' + ( ', R2 检查中' if nsh and done >= nsh else '' )}
if st == 'train' :
tag , bs = info . get ( 'tag' , 'base' ), info . get ( 'bench_steps' , 2000 )
b , m = last_rec ( f 'bench_ { tag } ' ) if newer ( f ' { RUNS } /bench_ { tag } /log.jsonl' , t0 ) else None , last_rec ( tag ) if newer ( f ' { RUNS } / { tag } /log.jsonl' , t0 ) else None
if m and m . get ( 'total' ):
return { 'frac' : ( bs + m [ 'step' ]) / ( bs + m [ 'total' ]), 'label' : f '正式训练 { m [ "step" ] : , } / { m [ "total" ] : , } 步' , 'run' : tag }
if b : return { 'frac' : b [ 'step' ] / ( bs + 60000 ), 'label' : f '测速 { b [ "step" ] : , } / { bs : , } 步' , 'run' : f 'bench_ { tag } ' }
return { 'frac' : 0 , 'label' : '加载数据中' }
if st == 'apg' :
r = last_rec ( 'apg' ) if newer ( f ' { RUNS } /apg/log.jsonl' , t0 ) else None
return { 'frac' : r [ 'it' ] / r [ 'total' ] if r and r . get ( 'total' ) else 0 , 'label' : f 'APG { r [ "it" ] : , } / { r [ "total" ] : , } 步' if r else '准备中' , 'run' : 'apg' }
if st == 'mine' :
R = info . get ( 'rounds' , 6 )
k = len ( newer ( f ' { RUNS } /mine/r*/gate.json' , t0 ))
cur = sorted ( newer ( f ' { RUNS } /apg_r*/log.jsonl' , t0 ), key = os . path . getmtime )
frac_in , lab = 0 , f '第 { k + 1 } 轮:挖错出题'
if cur and len ( cur ) > k :
name = os . path . basename ( os . path . dirname ( cur [ - 1 ])); r = last_rec ( name )
if r and r . get ( 'total' ): frac_in = 0.3 + 0.6 * r [ 'it' ] / r [ 'total' ]; lab = f '第 { k + 1 } 轮:APG { r [ "it" ] : , } / { r [ "total" ] : , } 步'
return { 'frac' : min ( 1 , ( k + frac_in ) / R ), 'label' : f ' { lab } (共最多 { R } 轮,提升不足会提前停)' , 'run' : name }
return { 'frac' : min ( 1 , k / R ), 'label' : f ' { lab } (共最多 { R } 轮,提升不足会提前停)' }
if st == 'm5' :
tag = info . get ( 'tag' , 'base' )
n = len ( newer ( f ' { RUNS } /m5_ { tag } _policy.json' , t0 )) + len ( newer ( f ' { RUNS } /m5_ { tag } _safety.json' , t0 ))
return { 'frac' : n / 2 , 'label' : [ '纯策略评估中' , '加安全层评估中' , '汇总' ][ min ( n , 2 )]}
return { 'frac' : None , 'label' : '进行中' }
def pipeline ():
d = jload ( os . path . join ( ROOT , RUNS , 'pipeline.json' ))
if not d : return None
cur = d . get ( 'current' )
if cur and cur in d [ 'stages' ]:
try : d [ 'progress' ] = stage_progress ( cur , d [ 'stages' ][ cur ])
except Exception as e : d [ 'progress' ] = { 'frac' : None , 'label' : f '进度读取失败: { e } ' }
return d
def status ():
P = lambda * a : os . path . join ( ROOT , * a )
mine = []
for d in sorted ( glob . glob ( P ( RUNS , 'mine' , 'r*' )), key = lambda x : int ( re . sub ( r '\D' , '' , os . path . basename ( x )) or 0 )):
rep , gate = jload ( os . path . join ( d , 'report.json' )), jload ( os . path . join ( d , 'gate.json' ))
if rep or gate :
mine . append ({ 'round' : os . path . basename ( d ), 'report' : rep and { k : rep . get ( k ) for k in ( 'n' , 'E1' , 'E2' , 'E3' , 'E4' , 'E5' , 'ok' , 'pool' )},
'gate' : gate and { k : gate . get ( k ) for k in ( 'gain_pp' , 'A_drop_pp' , 'fixed_kept' , 'R7_left_share_shift_pp' , 'R7_travel_median_ratio' , 'accept' )}})
stats = jload ( P ( DATA , 'stats.json' ))
if stats : stats = { k : stats . get ( k ) for k in ( 'tasks' , 'keep' , 'gapDrop' , 'minGapFilterMm' , 'keepRate' , 'samplesEvery3' , 'samplesPerTask' , 'projectedSamplesFor50k' , 'consensusDrop' , 'vertexMinGapMm' , 'execS' , 'travelDeg' )}
shards = glob . glob ( P ( DATA , 'shard_*.jsonl' ))
2026-10-17 18:57:48 +00:00
pl = pipeline ()
if pl and pl . get ( 'current' ) == 'gen' : # 只在生成阶段数行(增量);其他阶段用 stats.json 的 tasks
gen_lines = sum ( LC . count ( f ) for f in shards + glob . glob ( P ( DATA , 'shard_*.jsonl.part' )))
else : gen_lines = ( stats or {}) . get ( 'tasks' ) or 0
2026-10-17 17:15:16 +00:00
pb = jload ( P ( RUNS , 'policy_bench.json' ))
if pb : pb = { k : { kk : v . get ( kk ) for kk in ( 'n' , 'ok' , 'reached' , 'collisionFrames' , 'brakeRate' , 'recoveryRate' , 'livelock' , 'inferMsP99' , 'safetyMsP99' )}
for k , v in pb . items () if isinstance ( v , dict ) and 'n' in v }
2026-10-17 18:57:48 +00:00
return { 'now' : time . time (), 'gpu' : gpu (), 'pipeline' : pl , 'check' : CHECK . get (), 'lock' : lock_info (),
'stage_meta' : [{ 'id' : k , 'name' : n , 'est_h' : h , 'timeout_h' : pstate . TIMEOUT_H [ k ]} for k , n , h in pstate . STAGES ],
'r4' : jload ( P ( RUNS , 'r4.json' )),
2026-10-17 17:15:16 +00:00
'data' : { 'consistency' : ( jload ( P ( DATA , 'consistency.json' )) or {}) . get ( 'consistentByGateRule' ),
'gen_tasks_written' : gen_lines , 'shards' : len ( shards ), 'stats' : stats ,
'index' : jload ( P ( DATA , 'samples' , 'index.json' )), 'r2' : jload ( P ( DATA , 'r2.json' ))},
'm5' : [ x for x in ( jload ( f ) for f in sorted ( glob . glob ( P ( RUNS , 'm5_gate_*.json' )))) if x ],
'selected' : open ( P ( RUNS , 'selected_ckpt' )) . read () . strip () if os . path . exists ( P ( RUNS , 'selected_ckpt' )) else None ,
'mine' : mine , 'bench' : pb , 'acceptance' : jload ( P ( RUNS , 'acceptance.json' ))}
2026-10-17 18:57:48 +00:00
STAGES = pstate . ORDER
ENV_KEYS = { 'N_TASKS' : r '\d+' , 'NPROC' : r '\d+' , 'TARGET_SAMPLES' : r '\d+' , 'CFG' : r 'ml/configs/(base|wta|small)\.json' ,
'CONSENSUS' : r '[01]?' , 'ROUNDS' : r '\d+' , 'APG_STEPS' : r '\d+' , 'MINE_N' : r '\d+' , 'MIN_GAP_MM' : r '\d+(\.\d+)?' ,
'FRESH' : r '[01]' , 'R2_STRICT' : r '[01]' , 'AUTO_WTA' : r '[01]' }
2026-10-17 17:15:16 +00:00
CONSOLE = os . path . join ( ROOT , RUNS , 'console.log' )
def find_bash ():
"""Windows 优先 Git Bash( System32 \\ bash.exe 是 WSL,不能用);其他系统用 PATH 里的 bash。"""
if os . name == 'nt' :
for p in [ os . environ . get ( 'BASH' , '' ), r 'C:\Program Files\Git\bin\bash.exe' , r 'C:\Program Files (x86)\Git\bin\bash.exe' ,
os . path . expandvars ( r '%LOCALAPPDATA%\Programs\Git\bin\bash.exe' )]:
if p and os . path . exists ( p ): return p
g = shutil . which ( 'git' )
if g :
p = os . path . join ( os . path . dirname ( os . path . dirname ( g )), 'bin' , 'bash.exe' )
if os . path . exists ( p ): return p
return None
return shutil . which ( 'bash' )
2026-10-17 18:57:48 +00:00
def train_python ():
"""流水线用的 Python:尊重 PYTHON 环境变量(与 console.sh / console.bat 一致),否则用控制台自己的解释器。"""
return os . environ . get ( 'PYTHON' ) or sys . executable
class SelfCheck :
"""启动自检(审查 N-10):在后台线程跑一次,结果缓存;「开始训练」前不过就拒绝,网页顶栏显示。"""
def __init__ ( self ): self . r = { 'pending' : True }; threading . Thread ( target = self . run , daemon = True ) . start ()
def run ( self ):
py = train_python (); r = { 'python' : py , 'bash' : find_bash (), 'node' : shutil . which ( 'node' )}
try :
o = subprocess . run ([ py , '-c' , 'import json,torch;c=torch.cuda.is_available();print(json.dumps({"torch":torch.__version__,"cuda":c,"gpu":torch.cuda.get_device_name(0) if c else None}))' ],
capture_output = True , text = True , timeout = 120 )
r . update ( json . loads ( o . stdout . strip () . splitlines ()[ - 1 ]) if o . returncode == 0 else { 'torch' : None , 'torch_error' : ( o . stderr or o . stdout ) . strip ()[ - 300 :]})
except Exception as e : r . update ( torch = None , torch_error = str ( e ))
probs = []
if not r . get ( 'torch' ): probs . append ( f '这个 Python 没有 torch: { py } (先激活训练环境再启动控制台,或设 PYTHON=...) ' )
if not r [ 'bash' ]: probs . append ( '找不到 bash: Windows 请安装 Git for Windows(自带 Git Bash) ' )
if not r [ 'node' ]: probs . append ( '找不到 node' )
if r . get ( 'torch' ) and not r . get ( 'cuda' ): r [ 'warn' ] = 'torch 看不到 GPU,训练会极慢'
r [ 'problems' ] = probs ; r [ 'ok' ] = not probs
self . r = r
def get ( self ): return self . r
def lock_info ():
lk = pstate . lock_read ()
if not lk : return None
return dict ( lk , alive = pstate . lock_alive ( lk ), age_s = time . time () - lk . get ( 'beat' , 0 ))
def kill_session ( sid ):
"""POSIX:结束会话 sid 里的所有进程(含 timeout 另起的进程组)。"""
pids = []
for d in os . listdir ( '/proc' ) if os . path . isdir ( '/proc' ) else []:
if d . isdigit ():
try :
if os . getsid ( int ( d )) == sid : pids . append ( int ( d ))
except Exception : pass
if not pids :
try : os . killpg ( sid , signal . SIGTERM )
except Exception : pass
return
for p in pids :
try : os . kill ( p , signal . SIGTERM )
except Exception : pass
2026-10-17 17:15:16 +00:00
class Job :
def __init__ ( self ):
self . p = None ; self . info = {}; self . lock = threading . Lock ()
def state ( self ):
with self . lock :
run = self . p is not None and self . p . poll () is None
if self . p is not None and not run and self . info . get ( 'rc' ) is None :
self . info [ 'rc' ] = self . p . returncode ; self . info [ 'ended' ] = time . time ()
2026-10-17 18:57:48 +00:00
lk = lock_info ()
ext = bool ( lk and lk [ 'alive' ]) and not run # 终端或上一个控制台窗口起的流水线
return { 'running' : run or ext , 'external' : ext , 'lock' : lk , ** self . info , 'bash' : find_bash () is not None }
2026-10-17 17:15:16 +00:00
def start ( self , frm , to , env ):
with self . lock :
if self . p is not None and self . p . poll () is None : return { 'error' : '已经在运行' }
2026-10-17 18:57:48 +00:00
lk = lock_info ()
if lk and lk [ 'alive' ]:
return { 'error' : f "已有一条流水线在运行( { lk . get ( 'owner' ) } , pid { lk . get ( 'pid' ) } , { lk [ 'age_s' ] : .0f } s 前心跳);先停止它" }
ck = CHECK . get ()
if ck . get ( 'pending' ): return { 'error' : '环境自检还没完成,稍等几秒再点' }
if not ck . get ( 'ok' ): return { 'error' : '环境自检未通过:' + '; ' . join ( ck [ 'problems' ])}
2026-10-17 17:15:16 +00:00
if frm not in STAGES or to not in STAGES or STAGES . index ( frm ) > STAGES . index ( to ): return { 'error' : '起止阶段无效' }
bash = find_bash ()
2026-10-17 20:28:00 +00:00
e = dict ( os . environ , FROM = frm , TO = to , MONITOR = '0' , PYTHONUNBUFFERED = '1' , PYTHONIOENCODING = 'utf-8' , PYTHONUTF8 = '1' , PIPE_OWNER = '网页' )
2026-10-17 18:57:48 +00:00
py = train_python ()
if ' ' not in py : e [ 'PYTHON' ] = py . replace ( ' \\ ' , '/' )
2026-10-17 17:15:16 +00:00
for k , v in ( env or {}) . items ():
if k in ENV_KEYS and v not in ( None , '' ) and re . fullmatch ( ENV_KEYS [ k ], str ( v )): e [ k ] = str ( v )
os . makedirs ( os . path . dirname ( CONSOLE ), exist_ok = True )
2026-10-17 18:57:48 +00:00
shown = ' ' . join ( f ' { k } = { e [ k ] } ' for k in ( 'FRESH' , 'FROM' , 'TO' ) if k in e )
with open ( CONSOLE , 'w' , encoding = 'utf-8' ) as f : f . write ( f '$ { shown } bash ml/pipeline.sh all [ { time . strftime ( "%Y-%m- %d %H:%M:%S" ) } ] \n ' )
2026-10-17 17:15:16 +00:00
log = open ( CONSOLE , 'a' , encoding = 'utf-8' ) # 追加模式:网页写入的"已停止"提示不会被子进程覆盖
2026-10-17 18:57:48 +00:00
# Windows: CREATE_NO_WINDOW 给子进程树一个隐藏的独立控制台 → 关掉控制台黑窗口不会连带杀训练(审查 N-11)
kw = { 'creationflags' : subprocess . CREATE_NEW_PROCESS_GROUP | 0x08000000 } if os . name == 'nt' else { 'start_new_session' : True }
2026-10-17 17:15:16 +00:00
self . p = subprocess . Popen ([ bash , 'ml/pipeline.sh' , 'all' ], cwd = ROOT , env = e , stdout = log , stderr = subprocess . STDOUT , stdin = subprocess . DEVNULL , ** kw )
self . info = { 'pid' : self . p . pid , 'from' : frm , 'to' : to , 'started' : time . time (), 'rc' : None , 'ended' : None ,
'env' : { k : e [ k ] for k in ENV_KEYS if k in e }}
return { 'ok' : True , 'pid' : self . p . pid }
def stop ( self ):
2026-10-17 18:57:48 +00:00
# 2026-10-18 体面停止:训练类阶段先写 ml/runs/STOP,等训练进程自己存 last.pt 并退出(最多 90 s),再硬杀剩下的。
# Windows 的 taskkill /F 收不到信号,不这样做会丢掉上次存档之后的进度。生成/打包本身按 shard/条断点续,直接杀。
with self . lock :
ours = self . p is not None and self . p . poll () is None
lk = lock_info ()
if not ours and not ( lk and lk [ 'alive' ]): return { 'error' : '没有在运行' }
pid = self . p . pid if ours else int ( lk . get ( 'pid' ) or 0 )
if not pid : return { 'error' : '锁里没有 pid,无法停止' }
cur = ( pstate . load () . get ( 'current' ) or '' )
if cur in ( 'train' , 'm5' , 'apg' , 'mine' ):
open ( STOPF , 'w' ) . write ( time . strftime ( '%H:%M:%S' ))
with open ( CONSOLE , 'a' , encoding = 'utf-8' ) as f : f . write ( f ' \n [停止请求已发出,等待 { cur } 存档 last.pt…] \n ' )
for _ in range ( 180 ):
time . sleep ( 0.5 )
l2 = pstate . lock_read ()
if ( self . p is not None and self . p . poll () is not None ) if ours else not ( l2 and pstate . lock_alive ( l2 )): break
2026-10-17 17:15:16 +00:00
with self . lock :
if os . name == 'nt' :
2026-10-17 18:57:48 +00:00
subprocess . run ([ 'taskkill' , '/T' , '/F' , '/PID' , str ( pid )], capture_output = True )
2026-10-17 17:15:16 +00:00
else :
2026-10-17 18:57:48 +00:00
if ours :
kill_session ( pid )
try : self . p . wait ( 15 )
except subprocess . TimeoutExpired : kill_session ( pid ); self . p . kill ()
else : # 只动它自己的会话 / 进程组,绝不按会话杀到用户的终端
try :
if os . getsid ( pid ) == pid : kill_session ( pid )
elif os . getpgid ( pid ) == pid : os . killpg ( pid , signal . SIGTERM )
else : os . kill ( pid , signal . SIGTERM )
except ProcessLookupError : pass
2026-10-17 17:15:16 +00:00
self . info [ 'stopped' ] = True
with open ( CONSOLE , 'a' , encoding = 'utf-8' ) as f : f . write ( f ' \n [已在网页上停止 { time . strftime ( "%H:%M:%S" ) } ] \n ' )
2026-10-17 18:57:48 +00:00
time . sleep ( 0.5 )
pstate . mark_stopped ( - 15 ) # 当前阶段记为"已停止"、释放锁(单一写者)
try : os . remove ( STOPF )
except FileNotFoundError : pass
2026-10-17 17:15:16 +00:00
return { 'ok' : True }
2026-10-17 18:57:48 +00:00
STOPF = os . path . join ( ROOT , 'ml' , 'runs' , 'STOP' ) # 与 ml/common/ckpt.py 的 STOP 同一文件
2026-10-17 17:15:16 +00:00
JOB = Job ()
2026-10-17 18:57:48 +00:00
CHECK = None # main() 里创建(自检要等 find_bash 定义完)
2026-10-17 17:15:16 +00:00
def console ( offset ):
if not os . path . exists ( CONSOLE ): return { 'text' : '' , 'offset' : 0 , 'reset' : True }
size = os . path . getsize ( CONSOLE ); reset = offset > size or offset == 0
if reset : offset = max ( 0 , size - 65536 ) # 首次只给最后 64 KB
with open ( CONSOLE , 'rb' ) as f : f . seek ( offset ); b = f . read ( min ( size - offset , 1 << 20 ))
return { 'text' : b . decode ( 'utf-8' , 'replace' ) . replace ( ' \r ' , '' ), 'offset' : offset + len ( b ), 'reset' : reset }
class H ( BaseHTTPRequestHandler ):
def log_message ( self , * a ): pass
def send ( self , code , body , ctype ):
b = body if isinstance ( body , bytes ) else body . encode ( 'utf-8' )
self . send_response ( code ); self . send_header ( 'Content-Type' , ctype ); self . send_header ( 'Content-Length' , str ( len ( b )))
self . send_header ( 'Cache-Control' , 'no-store' ); self . end_headers (); self . wfile . write ( b )
def do_GET ( self ):
u = urlparse ( self . path ); q = parse_qs ( u . query )
try :
if u . path in ( '/' , '/index.html' ):
with open ( HTML , 'rb' ) as f : return self . send ( 200 , f . read (), 'text/html; charset=utf-8' )
if u . path == '/api/runs' : return self . send ( 200 , json . dumps ( list_runs ()), 'application/json' )
if u . path == '/api/log' :
return self . send ( 200 , json . dumps ( read_log ( q . get ( 'run' , [ '' ])[ 0 ], int ( q . get ( 'offset' , [ '0' ])[ 0 ]))), 'application/json' )
if u . path == '/api/status' : return self . send ( 200 , json . dumps ( status ()), 'application/json' )
if u . path == '/api/job' : return self . send ( 200 , json . dumps ( JOB . state ()), 'application/json' )
if u . path == '/api/console' : return self . send ( 200 , json . dumps ( console ( int ( q . get ( 'offset' , [ '0' ])[ 0 ]))), 'application/json' )
self . send ( 404 , 'not found' , 'text/plain' )
except Exception as e :
self . send ( 500 , json . dumps ({ 'error' : str ( e )}), 'application/json' )
def do_POST ( self ):
u = urlparse ( self . path )
if not ( self . server . remote_control or self . client_address [ 0 ] in ( '127.0.0.1' , '::1' )):
return self . send ( 403 , json . dumps ({ 'error' : '开始/停止只接受本机操作' }), 'application/json' )
2026-10-17 18:57:48 +00:00
# 审查 N-8:本机任意网页都能向 localhost 发 POST(源地址同为 127.0.0.1)。要求自定义头(跨站请求会触发 CORS 预检,
# 本服务不应答 OPTIONS → 浏览器不发),并且 Origin / Sec-Fetch-Site 若存在必须同源。
org , sfs , host = self . headers . get ( 'Origin' ), self . headers . get ( 'Sec-Fetch-Site' ), self . headers . get ( 'Host' , '' )
if self . headers . get ( 'X-Console' ) != '1' or ( sfs and sfs not in ( 'same-origin' , 'none' )) or \
( org and urlparse ( org ) . netloc not in ( host , f 'localhost: { self . server . server_port } ' , f '127.0.0.1: { self . server . server_port } ' )):
return self . send ( 403 , json . dumps ({ 'error' : '拒绝:非本控制台页面发起的请求' }), 'application/json' )
2026-10-17 17:15:16 +00:00
try :
n = int ( self . headers . get ( 'Content-Length' ) or 0 ); body = json . loads ( self . rfile . read ( n ) or b ' {} ' )
2026-10-17 18:57:48 +00:00
if u . path == '/api/start' :
env = dict ( body . get ( 'env' ) or {})
if body . get ( 'fresh' ): env [ 'FRESH' ] = '1'
r = JOB . start ( body . get ( 'from' , 'kin' ), body . get ( 'to' , 'accept' ), env )
2026-10-17 17:15:16 +00:00
elif u . path == '/api/stop' : r = JOB . stop ()
else : return self . send ( 404 , 'not found' , 'text/plain' )
self . send ( 200 if 'ok' in r else 409 , json . dumps ( r , ensure_ascii = False ), 'application/json' )
except Exception as e :
self . send ( 500 , json . dumps ({ 'error' : str ( e )}), 'application/json' )
def main ():
ap = argparse . ArgumentParser ()
ap . add_argument ( '--port' , type = int , default = 8100 ); ap . add_argument ( '--host' , default = '127.0.0.1' )
ap . add_argument ( '--open' , action = 'store_true' , help = '启动后自动打开浏览器' )
ap . add_argument ( '--remote-control' , action = 'store_true' , help = '允许其他机器点开始/停止(默认只能看)' )
a = ap . parse_args ()
2026-10-17 18:57:48 +00:00
global CHECK
CHECK = SelfCheck ()
2026-10-17 17:15:16 +00:00
srv = ThreadingHTTPServer (( a . host , a . port ), H ); srv . remote_control = a . remote_control
url = f 'http://localhost: { a . port } /'
2026-10-17 18:57:48 +00:00
print ( f '训练控制台: { url } (关闭本窗口只关控制台,已开始的训练继续跑;重新打开控制台即可接着看、可停止)' , flush = True )
2026-10-17 17:15:16 +00:00
if a . open : threading . Timer ( 0.8 , lambda : webbrowser . open ( url )) . start ()
try : srv . serve_forever ()
except KeyboardInterrupt : pass
if __name__ == '__main__' :
main ()