基于openEuler 24.03 实现 DeepSeek-R1:671B 的高效分布式训练

# 安装必要依赖
sudo dnf install -y kernel-devel kernel-headers gcc make cmake git python3-devel
sudo dnf groupinstall -y "Development Tools"


# 禁用防火墙和SELinux（生产环境需谨慎）
sudo systemctl stop firewalld
sudo systemctl disable firewalld
sudo setenforce 0
sudo sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' etc/selinux/config


# 配置SSH免密登录（多节点训练必备）
ssh-keygen -t rsa
ssh-copy-id user@node1
ssh-copy-id user@node2
...

# 检查内核版本（需与驱动兼容）
uname -r  # 例如：5.15.0-101.oe2403.x86_64


# 禁用nouveau驱动
echo "blacklist nouveau" | sudo tee etc/modprobe.d/blacklist-nouveau.conf
sudo dracut --force


# 下载并安装驱动（选择CUDA 12.2+兼容版本）
wget https://us.download.nvidia.com/tesla/535.129.03/NVIDIA-Linux-x86_64-535.129.03.run
sudo sh NVIDIA-Linux-x86_64-535.129.03.run --silent --dkms


# 验证驱动
nvidia-smi  # 应显示20张A100

# 安装CUDA Toolkit
wget https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda_12.2.2_535.104.05_linux.run
sudo sh cuda_12.2.2_535.104.05_linux.run --silent --toolkit


# 安装cuDNN和NCCL（需从NVIDIA官网下载）
tar -xzf cudnn-linux-x86_64-8.9.6.50_cuda12-archive.tar.xz
sudo cp -r cudnn-*-archive/include/* usr/local/cuda/include/
sudo cp -r cudnn-*-archive/lib/* usr/local/cuda/lib64/


tar -xzf nccl_2.18.5-1+cuda12.2_x86_64.txz
sudo cp -r nccl_*/lib/* usr/local/cuda/lib64/
sudo cp nccl_*/include/* usr/local/cuda/include/


# 添加环境变量
echo 'export PATH=/usr/local/cuda/bin:$PATH' >> ~/.bashrc
echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH' >> ~/.bashrc
source ~/.bashrc

# 创建Python虚拟环境
python3 -m venv deepseek-env
source deepseek-env/bin/activate


# 安装PyTorch与DeepSpeed
pip install torch==2.2.0+cu121 torchvision==0.17.0+cu121 torchaudio==2.2.0+cu121 --index-url https://download.pytorch.org/whl/cu121
pip install deepspeed==0.13.0 ninja


# 验证GPU支持
python -c "import torch; print(torch.cuda.device_count())"  # 应输出20

# 安装Mellanox OFED驱动（适配openEuler）
wget https://content.mellanox.com/ofed/MLNX_OFED-24.01-0.5.6.0/MLNX_OFED_LINUX-24.01-0.5.6.0-rhel9.2-x86_64.tgz
tar -xzf MLNX_OFED-*.tgz
cd MLNX_OFED_LINUX-*-rhel9.2-x86_64
sudo ./mlnxofedinstall --without-fw-update --force
sudo etc/init.d/openibd restart


# 配置NCCL参数
echo 'export NCCL_IB_HCA=mlx5' >> ~/.bashrc
echo 'export NCCL_IB_GID_INDEX=3' >> ~/.bashrc
echo 'export NCCL_SOCKET_IFNAME=ib0' >> ~/.bashrc
echo 'export NCCL_DEBUG=WARN' >> ~/.bashrc
source ~/.bashrc

# 克隆DeepSeek官方仓库（假设已授权）
git clone https://github.com/deepseek-ai/DeepSeek-R1
cd DeepSeek-R1


# 下载预训练权重与数据集
wget https://models.deepseek.com/deepseek-r1-671b.tar.gz
tar -xzf deepseek-r1-671b.tar.gz


# 数据集预处理（需根据实际数据调整）
python tools/preprocess_data.py --input data/raw_text.jsonl --output-prefix my_dataset

# 创建hostfile（假设节点为node1-node4）
cat > hostfile << EOF
node1 slots=5
node2 slots=5
node3 slots=5
node4 slots=5
EOF


# 使用DeepSpeed启动训练
deepspeed --hostfile hostfile \
  --master_addr node1 \
  --launcher openmpi \
  --num_gpus 5 \
  train.py \
  --model_config configs/671b.yaml \
  --train_data my_dataset \
  --deepspeed_config ds_config.json \
  --batch_size 8 \
  --gradient_accumulation_steps 16

{
  "train_micro_batch_size_per_gpu": 1,
  "gradient_accumulation_steps": 16,
  "optimizer": {
    "type": "AdamW",
    "params": {
      "lr": 1e-5,
      "weight_decay": 0.01
    }
  },
  "zero_optimization": {
    "stage": 3,
    "offload_optimizer": {
      "device": "cpu",
      "pin_memory": true
    },
    "allgather_partitions": true,
    "allgather_bucket_size": 5e8,
    "overlap_comm": true
  },
  "fp16": {
    "enabled": true,
    "loss_scale_window": 1000
  },
  "flops_profiler": {
    "enabled": true,
    "profile_step": 10
  }
}

# 锁定GPU频率至最高性能
sudo nvidia-smi -lgc 1410,1410  # A100默认峰值频率


# 启用持久化模式
sudo nvidia-smi -pm 1


# 启用MPS（多进程服务）
sudo nvidia-cuda-mps-control -d

# 在模型代码中添加（减少内存碎片）
torch.cuda.set_per_process_memory_fraction(0.9)


# 启用激活检查点（Activation Checkpointing）
from torch.utils.checkpoint import checkpoint
def forward(self, x):
    return checkpoint(self._forward_impl, x)

# 调整网络缓冲区（InfiniBand优化）
echo "net.core.rmem_max=2147483647" | sudo tee -a /etc/sysctl.conf
echo "net.core.wmem_max=2147483647" | sudo tee -a /etc/sysctl.conf


# 提升文件句柄限制
echo "* soft nofile 1048576" | sudo tee -a /etc/security/limits.conf
echo "* hard nofile 1048576" | sudo tee -a /etc/security/limits.conf


# 应用配置
sudo sysctl -p

# GPU监控（跨节点）
nvidia-smi --query-gpu=timestamp,name,utilization.gpu,utilization.memory --format=csv -l 5


# NCCL通信分析
export NCCL_DEBUG=INFO
export NCCL_DEBUG_SUBSYS=COLL


# 带宽测试（节点间）
ib_write_bw -d mlx5_0 -F --report_gbits

export NCCL_IB_TIMEOUT=22
export NCCL_IB_RETRY_CNT=7

"offload_optimizer": { "device": "nvme", "nvme_path": "/mnt/nvme" }

nsys profile -w true -o deepseek_profile --capture-range=cudaProfilerApi \
--stop-on-range-end=true --cudabacktrace=true python train.py