forked from jiuyuan/CPM-9G-8B
61 lines
1.7 KiB
Bash
61 lines
1.7 KiB
Bash
|
#! /bin/bash
|
|||
|
|
#SBATCH --nodes=1
|
|||
|
|
#SBATCH --ntasks-per-node=8
|
|||
|
|
#SBATCH --gres=gpu:8
|
|||
|
|
#SBATCH --cpus-per-task=8
|
|||
|
|
#SBATCH --mem=512GB
|
|||
|
|
|
|||
|
|
# use 8 GPU for example, pretrain may need 32 GPU
|
|||
|
|
export MASTER_ADDR=`hostname`
|
|||
|
|
export MASTER_PORT=12345
|
|||
|
|
|
|||
|
|
mkdir -p /home/${USERNAME}/logs/debug
|
|||
|
|
mkdir -p /home/${USERNAME}/logs/tensorboard/cpm9g/
|
|||
|
|
|
|||
|
|
cd apps/cpm9g
|
|||
|
|
CONFIG_NAME="config/11b"
|
|||
|
|
# --------------- 运行参数 ---------------
|
|||
|
|
OPTS=""
|
|||
|
|
OPTS+=" --model-config ${CONFIG_NAME}/config.json"
|
|||
|
|
OPTS+=" --vocab ${CONFIG_NAME}/vocab.txt"
|
|||
|
|
OPTS+=" --batch-size 4"
|
|||
|
|
OPTS+=" --train-iters 400000"
|
|||
|
|
OPTS+=" --save-iters 250"
|
|||
|
|
OPTS+=" --save-name cpm9g_checkpoint"
|
|||
|
|
OPTS+=" --max-length 4096"
|
|||
|
|
OPTS+=" --lr 1.5e-5"
|
|||
|
|
OPTS+=" --inspect-iters 100"
|
|||
|
|
OPTS+=" --warmup-iters 2000"
|
|||
|
|
OPTS+=" --lr-decay-style noam"
|
|||
|
|
OPTS+=" --weight-decay 0.1"
|
|||
|
|
OPTS+=" --clip-grad 1.0"
|
|||
|
|
OPTS+=" --loss-scale 1048576"
|
|||
|
|
OPTS+=" --loss-scale-steps 32"
|
|||
|
|
OPTS+=" --offload"
|
|||
|
|
OPTS+=" --flash cuda"
|
|||
|
|
# OPTS+=" --load-grad"
|
|||
|
|
|
|||
|
|
# --------------- 写文件路径 ---------------
|
|||
|
|
## checkpoint
|
|||
|
|
OPTS+=" --save /home/${USERNAME}/checkpoints/cpm9g/"
|
|||
|
|
OPTS+=" --save-model /home/${USERNAME}/models/cpm9g/"
|
|||
|
|
|
|||
|
|
## logs，/local/logs 等价于 /data/logs（软链）
|
|||
|
|
OPTS+=" --log-dir /home/${USERNAME}/logs/train/"
|
|||
|
|
OPTS+=" --tensorboard /home/${USERNAME}/tensorboard/cpm9g/"`date +"%Y%m%d%H%M%S"`
|
|||
|
|
|
|||
|
|
# --------------- 读文件路径 ---------------
|
|||
|
|
OPTS+=" --dataset config/datasets.json"
|
|||
|
|
OPTS+=" --load ${CHECKPOINT}"
|
|||
|
|
OPTS+=" --start-step 1"
|
|||
|
|
|
|||
|
|
# --------------- 透传参数 ---------------
|
|||
|
|
OPTS+=" $@"
|
|||
|
|
|
|||
|
|
# --------------- 最终指令 ---------------
|
|||
|
|
|
|||
|
|
CMD="torchrun --nnodes=1 --nproc_per_node=8 --rdzv_id=1 --rdzv_backend=c10d --rdzv_endpoint=${MASTER_ADDR}:${MASTER_PORT} pretrain_cpm9g.py ${OPTS}"
|
|||
|
|
echo "${CMD}"
|
|||
|
|
|
|||
|
|
$CMD
|