第五轮：锁/续跑/超时/自动分支/R4 探针与 encode-once/控制台加固；阶段内断点续跑（gen 按条、prep 按 shard、训练与 APG 定时原子存档+体面停止）；分歧以实现方意见为准
This commit is contained in:
Hark
+17
-5
@@ -3,7 +3,9 @@
|
||||
* --consensus k：邻域共识过滤。起点不变、目标位置各扰动 5 mm 造 k 个邻居跑老师，原任务路径若与多数邻居
|
||||
* 不在同一绕行类（pathdev.sameClass），整条样本 keep=false（why=consensus）。成本约 (1+k) 倍。
|
||||
* 每行一条任务：采样 → 老师 → 距离下界认证过滤 → 局部时间律（50 Hz）→ 写出。
|
||||
* 认证不过的样本也写出（keep=false + 原因），供 stats 统计丢弃率；训练只用 keep=true。 */
|
||||
* 认证不过的样本也写出（keep=false + 原因），供 stats 统计丢弃率；训练只用 keep=true。
|
||||
* 2026-10-18 断点续写：第 i 条任务的随机数只由 (seed, i) 决定（sampler.reseed）；输出文件已存在时保留其中完整的行、
|
||||
* 截掉写了一半的最后一行，从下一条接着写 —— 生成几小时被中断，重跑只补剩下的，不重来、不重复。 */
|
||||
'use strict';
|
||||
const fs = require('fs'), path = require('path');
|
||||
const args = {}; for (let i = 2; i < process.argv.length; i += 2) args[process.argv[i].replace(/^--/, '')] = process.argv[i + 1];
|
||||
@@ -20,10 +22,20 @@ const cert = require('./cert.cjs')(S, {});
|
||||
const { timeParam } = require('./timelaw.cjs');
|
||||
const fp = S.modelFingerprint ? S.modelFingerprint() : null;
|
||||
fs.mkdirSync(path.dirname(OUT), { recursive: true });
|
||||
const fd = fs.openSync(OUT, 'w');
|
||||
let done = 0;
|
||||
if (fs.existsSync(OUT)) {
|
||||
const keep = [];
|
||||
for (const l of fs.readFileSync(OUT, 'utf8').split('\n')) { if (!l) continue; try { JSON.parse(l); keep.push(l); } catch (e) { break; } }
|
||||
done = Math.min(keep.length, NT);
|
||||
fs.writeFileSync(OUT, keep.slice(0, done).map(l => l + '\n').join(''));
|
||||
if (done) process.stderr.write(`[seed ${SEED}] 续写：已有 ${done}/${NT} 条，从第 ${done + 1} 条继续\n`);
|
||||
}
|
||||
const fd = fs.openSync(OUT, 'a');
|
||||
const taskSeed = i => (Math.imul(SEED, 2654435761) ^ Math.imul(i + 1, 40503) ^ 0x9e3779b9) >>> 0;
|
||||
const r3 = v => Math.round(v * 1000) / 1000;
|
||||
const t00 = performance.now();
|
||||
for (let i = 0; i < NT; i++) {
|
||||
for (let i = done; i < NT; i++) {
|
||||
sampler.reseed(taskSeed(i));
|
||||
const task = GIVEN ? Object.assign({ cat: 'E2' }, GIVEN[i], { mode: GIVEN[i].R ? 'pose' : 'pos' }) : sampler.sample();
|
||||
const t = teach(task);
|
||||
const rec = { id: 's' + String(SEED).padStart(4, '0') + '_' + String(i).padStart(6, '0'), cat: task.cat, mode: task.mode,
|
||||
@@ -53,7 +65,7 @@ for (let i = 0; i < NT; i++) {
|
||||
if (t.poseRelaxed && task.mode === 'pose') { rec.mode = 'pos'; rec.R = null; rec.demoted = true; rec.reached = t.err < 1; }
|
||||
if (rec.keep) { const tp = timeParam(t.pts); rec.traj = tp.traj.map(q => q.map(r3)); rec.exec_s = r3(tp.steps * 0.02); }
|
||||
fs.writeSync(fd, JSON.stringify(rec) + '\n');
|
||||
if ((i + 1) % 25 === 0) process.stderr.write(`[seed ${SEED}] ${i + 1}/${NT} ${((performance.now() - t00) / 1000 / (i + 1)).toFixed(2)} s/条\n`);
|
||||
if ((i + 1) % 25 === 0) process.stderr.write(`[seed ${SEED}] ${i + 1}/${NT} ${((performance.now() - t00) / 1000 / (i + 1 - done)).toFixed(2)} s/条\n`);
|
||||
}
|
||||
fs.closeSync(fd);
|
||||
process.stderr.write(`[seed ${SEED}] done ${NT} in ${((performance.now() - t00) / 1000).toFixed(0)} s\n`);
|
||||
process.stderr.write(`[seed ${SEED}] done ${NT}（本次 ${NT - done}） in ${((performance.now() - t00) / 1000).toFixed(0)} s\n`);
|
||||
|
||||
+29
-2
@@ -1,11 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# 并行生成：bash ml/gen/run_parallel.sh <每进程条数> <进程数> [起始种子=1] ["--consensus 4"]
|
||||
# 例：8 核机器 5 万条 → bash ml/gen/run_parallel.sh 6250 8
|
||||
# 2026-10-18 第五轮（审查 N-7 / N-16 / 缺口 1）：
|
||||
# - 每个种子先写 shard_XXXX.jsonl.part，写完才改名 → 中断/并发都不会留下被截断却看似完整的 shard；
|
||||
# - 已存在的 shard_XXXX.jsonl（写完的）直接跳过；没写完的 .part 由 gen_dataset 从断点续写（只补剩下的条数）；
|
||||
# - 逐个 wait 子进程并检查返回码（此前裸 `wait` 恒为 0，某个 node 崩了也会继续统计）；
|
||||
# - 每 60 s 往控制台打一行节拍（已写条数 + 各进程最新进度），网页「终端输出」可见。
|
||||
set -e
|
||||
N=${1:-500}; P=${2:-2}; S0=${3:-1}; EXTRA=${4:-}
|
||||
cd "$(dirname "$0")/../.."
|
||||
mkdir -p ml/data
|
||||
for ((i=0;i<P;i++)); do s=$((S0+i)); node ml/gen/gen_dataset.cjs --n $N --seed $s --out ml/data/shard_$(printf %04d $s).jsonl $EXTRA 2> ml/data/log_$s.txt & done
|
||||
wait
|
||||
pids=(); seeds=()
|
||||
for ((i=0;i<P;i++)); do
|
||||
s=$((S0+i)); f=ml/data/shard_$(printf %04d $s).jsonl
|
||||
if [ -f "$f" ]; then echo "[gen] 种子 $s 已完成，跳过"; continue; fi
|
||||
( node ml/gen/gen_dataset.cjs --n $N --seed $s --out "$f.part" $EXTRA 2>> ml/data/log_$s.txt && mv "$f.part" "$f" ) &
|
||||
pids+=($!); seeds+=($s)
|
||||
done
|
||||
t0=$(date +%s); tl=$t0
|
||||
while :; do
|
||||
alive=0; for p in "${pids[@]}"; do kill -0 $p 2>/dev/null && alive=$((alive+1)); done
|
||||
[ $alive = 0 ] && break
|
||||
sleep 5
|
||||
if [ $(( $(date +%s) - tl )) -ge 60 ]; then
|
||||
tl=$(date +%s)
|
||||
n=$(cat ml/data/shard_*.jsonl.part 2>/dev/null | wc -l)
|
||||
last=$(for s in "${seeds[@]}"; do tail -n1 ml/data/log_$s.txt 2>/dev/null; done | head -n 2 | tr '\n' ' ')
|
||||
echo "[gen 节拍] $(( (tl-t0)/60 )) 分钟 运行中 $alive/${#pids[@]} 进程 本批已写 $n 条 $last"
|
||||
fi
|
||||
done
|
||||
fail=0
|
||||
for k in "${!pids[@]}"; do
|
||||
if ! wait ${pids[$k]}; then echo "[gen] 种子 ${seeds[$k]} 失败，最后几行："; tail -n 5 ml/data/log_${seeds[$k]}.txt || true; fail=1; fi
|
||||
done
|
||||
[ $fail = 0 ] || exit 6
|
||||
node ml/gen/stats.cjs ml/data/shard_*.jsonl > ml/data/stats.json
|
||||
cat ml/data/stats.json
|
||||
|
||||
+10
-5
@@ -1,5 +1,8 @@
|
||||
/* 任务采样（方案 §4.3）：A 均匀 45% / B 直线被挡 30% / C 贴立柱 15% / D 目标不可达 10%；pos/pose 各半（D 只做 pos） */
|
||||
'use strict';
|
||||
/* 2026-10-18 第五轮（审查 N-16）：所有拒绝采样都有次数上限，超限抛错带原因（此前 for(;;) 在 collide.json 改动后可能静默死循环）。 */
|
||||
const CAP = 200000;
|
||||
function giveUp(what, n) { throw new Error(`采样失败：${what} 连续 ${n} 次未命中——多半是 collide.json / 关节限位变了，导致该类位形几乎不存在`); }
|
||||
module.exports = function makeSampler(S, seed) {
|
||||
const L = S.LIM, O = S.ORDER, w = S.wrap180;
|
||||
let st = (seed >>> 0) || 1; const rnd = () => { st = (st * 1664525 + 1013904223) >>> 0; return st / 4294967296; };
|
||||
@@ -8,19 +11,20 @@ module.exports = function makeSampler(S, seed) {
|
||||
const segOK = (a, b) => { const dd = d(a, b); const n = Math.max(1, Math.ceil(Math.max(...dd.map(Math.abs)))); for (let i = 1; i <= n; i++) if (!S.collideFree(a.map((v, k) => v + dd[k] * i / n))) return false; return true; };
|
||||
const hitsColumn = q => (S.collideDetail(q) || []).some(p => /立柱/.test(p.cnA + p.cnB));
|
||||
const nearColumn = q => { for (const k of [0, 1, 2]) for (const sg of [-4, 4]) { const p = q.slice(); p[k] += sg; if (hitsColumn(p)) return true; } return false; };
|
||||
const freeQ = () => { for (;;) { const q = rq(); if (S.collideFree(q)) return q; } };
|
||||
const freeQ = () => { for (let n = 0; n < CAP; n++) { const q = rq(); if (S.collideFree(q)) return q; } giveUp('无碰位形', CAP); };
|
||||
function sample() {
|
||||
const u = rnd();
|
||||
const cat = u < 0.45 ? 'A' : u < 0.75 ? 'B' : u < 0.90 ? 'C' : 'D';
|
||||
let a, b, T, R = null;
|
||||
if (cat === 'A') { a = freeQ(); b = freeQ(); }
|
||||
else if (cat === 'B') { for (;;) { a = freeQ(); b = freeQ(); if (!segOK(a, b)) break; } }
|
||||
else if (cat === 'B') { let n = 0; for (;; n++) { if (n >= 20000) giveUp('B 类直线被挡', n); a = freeQ(); b = freeQ(); if (!segOK(a, b)) break; } }
|
||||
else if (cat === 'C') {
|
||||
for (;;) { b = rq(); if (!S.collideFree(b) || !nearColumn(b)) continue; a = null;
|
||||
let n = 0;
|
||||
for (;; n++) { if (n >= CAP) giveUp('C 类贴立柱终点', n); b = rq(); if (!S.collideFree(b) || !nearColumn(b)) continue; a = null;
|
||||
for (let t = 0; t < 200; t++) { const c = rq(); if (S.collideFree(c) && !segOK(c, b)) { a = c; break; } } if (a) break; }
|
||||
} else {
|
||||
a = freeQ();
|
||||
if (rnd() < 0.6) { for (;;) { b = rq(); if (hitsColumn(b)) break; } T = S.fk(b, 0); } // 目标位置落在立柱附近/内部
|
||||
if (rnd() < 0.6) { let n = 0; for (;; n++) { if (n >= CAP) giveUp('D 类立柱内目标', n); b = rq(); if (hitsColumn(b)) break; } T = S.fk(b, 0); } // 目标位置落在立柱附近/内部
|
||||
else { b = rq(); const p = S.fk(b, 0), o = S.fk(O.map(() => 0), 0); T = p.map((v, i) => o[i] + (v - o[i]) * 1.6); } // 工作空间外
|
||||
}
|
||||
const mode = cat === 'D' ? 'pos' : (rnd() < 0.5 ? 'pos' : 'pose');
|
||||
@@ -28,5 +32,6 @@ module.exports = function makeSampler(S, seed) {
|
||||
if (mode === 'pose') R = S.orient(b);
|
||||
return { cat, mode, q0: a, T, R, qb: b };
|
||||
}
|
||||
return { sample, rnd, segOK };
|
||||
const reseed = v => { st = (v >>> 0) || 1; }; // 2026-10-18：每条任务独立种子，shard 中断后可从第 i 条接着生成
|
||||
return { sample, rnd, segOK, reseed };
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user