#!/usr/bin/env bash
# 并行打包样本：bash ml/prep/run_prep.sh [进程数=CPU 核数]
set -e
cd "$(dirname "$0")/../.."
P=${1:-$(nproc)}
mkdir -p ml/data/samples
ls ml/data/shard_*.jsonl | xargs -P "$P" -I{} node ml/prep/build_dataset.cjs {} ml/data/samples
${PYTHON:-python} -m ml.data --index ml/data/samples