23 lines
1.0 KiB
Bash
Executable File
23 lines
1.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# 并行打包样本：bash ml/prep/run_prep.sh [进程数=CPU 核数]
|
|
# 2026-10-18 断点续跑：samples/<shard>.tasks.jsonl 存在且比 shard 新 → 该 shard 已打包完，跳过；
|
|
# shard 已不存在的旧打包结果删掉（不让上一批数据混进来）。中断后重跑只补没打完的 shard。
|
|
set -e
|
|
cd "$(dirname "$0")/../.."
|
|
P=${1:-$(nproc)}
|
|
D=ml/data/samples
|
|
mkdir -p $D
|
|
for t in $D/*.tasks.jsonl; do
|
|
[ -e "$t" ] || continue; b=$(basename "$t" .tasks.jsonl)
|
|
[ -f ml/data/$b.jsonl ] || { echo "[prep] 删除旧打包 $b（对应 shard 已不存在）"; rm -f $D/$b.*; }
|
|
done
|
|
todo=()
|
|
for f in ml/data/shard_*.jsonl; do
|
|
b=$(basename "$f" .jsonl)
|
|
if [ -f $D/$b.tasks.jsonl ] && [ $D/$b.tasks.jsonl -nt "$f" ]; then continue; fi
|
|
todo+=("$f")
|
|
done
|
|
echo "[prep] 共 $(ls ml/data/shard_*.jsonl | wc -l) 个 shard，待打包 ${#todo[@]} 个"
|
|
[ ${#todo[@]} = 0 ] || printf '%s\n' "${todo[@]}" | xargs -P "$P" -I{} node ml/prep/build_dataset.cjs {} $D
|
|
${PYTHON:-python} -m ml.data --index $D
|