9 lines
316 B
Bash
Executable File
9 lines
316 B
Bash
Executable File
#!/usr/bin/env bash
|
|
# 并行打包样本：bash ml/prep/run_prep.sh [进程数=CPU 核数]
|
|
set -e
|
|
cd "$(dirname "$0")/../.."
|
|
P=${1:-$(nproc)}
|
|
mkdir -p ml/data/samples
|
|
ls ml/data/shard_*.jsonl | xargs -P "$P" -I{} node ml/prep/build_dataset.cjs {} ml/data/samples
|
|
${PYTHON:-python} -m ml.data --index ml/data/samples
|