确保已安装docker

下载DeepSeek模型权重

apt-get install git-lfs 或者yum install git-lfs

cd /home

mkdir deepseek-ai

cd /home/deepseek-ai/

git clone https://www.modelscope.cn/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B.git

配置全向箔镜像地址

配置淄博habor的hosts vim /etc/hosts

120.220.95.189 zibo.harbor.iluvatar.com.cn

修改docker的配置，vim /etc/docker/daemon.json

{

"exec-opts": ["native.cgroupdriver=systemd"],

"insecure-registries": ["zibo.harbor.iluvatar.com.cn:30000"]

}

重启docker systemctl reload docker

拉取全向箔镜像

docker pull zibo.harbor.iluvatar.com.cn:30000/saas/mr-bi150-4.1.3-aarch64-ubuntu20.04-py3.10-poc-llm-infer:v1.2.1

启动容器

docker run -it -v /usr/src:/usr/src -v /lib/modules:/lib/modules -v /dev:/dev -v /home:/home -p 1000-1999:1000-1999 --name=test --pid=host --ipc=host --privileged --cap-add=ALL --pid=host zibo.harbor.iluvatar.com.cn:30000/saas/mr-bi150-4.1.3-aarch64-ubuntu20.04-py3.10-poc-llm-infer:v1.2.1 /bin/bash

运行offline_inference测试

cd /root/apps/llm-modelzoo/inference/Qwen/vllm
python3 offline_inference.py --model /home/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B/ --max-tokens 256 --max-model-len 2048 -tp 1 --temperature 0.55

api测试

cd ~/apps/llm-modelzoo/benchmark/vllm
# server 端

python3 -m vllm.entrypoints.openai.api_server --model /home/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B/ --gpu-memory-utilization 0.9 --max-num-batched-tokens 5120 --max-model-len 2048 --max-num-seqs 256 --host 0.0.0.0 --port 1234 --trust-remote-code

# client 端

python3 benchmark_serving_tokens.py --model /home/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B/ --host 0.0.0.0 --port 1234 --num-prompts 32 --input-tokens 256 --output-tokens 128 --time-interval 0.5 --trust-remote-code