前提已安装4.4.0驱动
如未安装SDK，请参考如下指南：
https://ixkb.iluvatar.com.cn:9443/webdoc/view/Pub8a16948a9a4cb023019cad9168850f71.html （mr系列）
或者
https://ixkb.iluvatar.com.cn:9443/webdoc/view/Pub8a16948a9a4cb023019cad814ae40f5d.html（天垓150系列） 进行安装
镜像下载
没有网的情况下可以sftp上下载放到服务器上
密码 联系天数售后工程师
sftp -P 29880 iluvatar_mr@iftp.iluvatar.com.cn
get /client_tmp/support/4.4/vllm0.17.0-4.4.0-x86.v6.tar /home
导入镜像
docker load -i /home/vllm0.17.0-4.4.0-x86.v6.tar
服务器能连外网情况下可以直接使用docker pull进行镜像拉取
docker pull registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v6
创建容器 (data是数据盘，根据实际数据盘进行调整）
docker run -dit -v /usr/src:/usr/src -v /lib/modules:/lib/modules -v /dev:/dev -v /home:/home -v /data:/data --network=host --name=Qwen --pid=host --ipc=host --privileged --cap-add=ALL registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v6 /bin/bash
模型下载（在容器内）
docker exec -it Qwen bash
mkdir -p /data/model
pip install modelscope
modelscope download --model iluvatar-corex/Qwen3.5-397B-A17B-W4A8 --local_dir ./Qwen3.5-397B-A17B-W4A8
modelscope download --model Qwen/Qwen3.5-122B-A10B --local_dir ./Qwen3.5-122B-A10B
modelscope download --model iluvatar-corex/Qwen3.5-122B-A10B-W4A8 --local_dir ./Qwen3.5-122B-A10B-W4A8
modelscope download --model Qwen/Qwen3.5-35B-A3B --local_dir ./Qwen3.5-35B-A3B
modelscope download --model iluvatar-corex/Qwen3.5-35B-A3B-W4A8 --local_dir ./Qwen3.5-35B-A3B-W4A8
modelscope download --model Qwen/Qwen3.5-27B --local_dir ./Qwen3.5-27B
modelscope download --model iluvatar-corex/Qwen3.5-27B-W8A8 --local_dir ./Qwen3.5-27B-W8A8
modelscope download --model Qwen/Qwen3.5-9B --local_dir ./Qwen3.5-9B
modelscope download --model iluvatar-corex/Qwen3.5-9B-W8A8 --local_dir ./Qwen3.5-9B-W8A8
部署模型（在容器内）
Qwen3.5-9B-W8A8
export VLLM_RPC_TIMEOUT=50000
export VLLM_ENFORCE_CUDA_GRAPH=1
export VLLM_KV_DISABLE_CROSS_GROUP_SHARE=1
vllm serve /data/model/Qwen3.5-9B-W8A8 --trust-remote-code -tp 1 --max-num-seqs 64 --enable-chunked-prefill --max-model-len 16384 --host 0.0.0.0 --port 1234 --gpu-memory-utilization 0.90 --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "level": 0}'
Qwen3.5-9B
export VLLM_RPC_TIMEOUT=50000
export VLLM_ENFORCE_CUDA_GRAPH=1
export VLLM_KV_DISABLE_CROSS_GROUP_SHARE=1
vllm serve /data/model/Qwen3.5-9B --trust-remote-code -tp 1 --max-num-seqs 64 --enable-chunked-prefill --max-model-len 16384 --host 0.0.0.0 --port 1234 --gpu-memory-utilization 0.90 --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "level": 0}'
Qwen3.5-27B-W8A8
export VLLM_RPC_TIMEOUT=50000
export VLLM_ENFORCE_CUDA_GRAPH=1
export VLLM_KV_DISABLE_CROSS_GROUP_SHARE=1
vllm serve /data/model/Qwen3.5-27B-W8A8 --trust-remote-code -tp 2 --max-num-seqs 64 --enable-chunked-prefill --max-model-len 16384 --host 0.0.0.0 --port 1234 --gpu-memory-utilization 0.90 --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "level": 0}'
Qwen3.5-27B
export VLLM_RPC_TIMEOUT=50000
export VLLM_ENFORCE_CUDA_GRAPH=1
export VLLM_KV_DISABLE_CROSS_GROUP_SHARE=1
vllm serve /data/model/Qwen3.5-27B --trust-remote-code -tp 4 --max-num-seqs 64 --enable-chunked-prefill --max-model-len 16384 --host 0.0.0.0 --port 1234 --gpu-memory-utilization 0.90 --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "level": 0}'
Qwen3.5-35B-A3B-W4A8
export VLLM_RPC_TIMEOUT=50000
export VLLM_ENFORCE_CUDA_GRAPH=1
export VLLM_W8A8_MOE_USE_W4A8=1
export VLLM_KV_DISABLE_CROSS_GROUP_SHARE=1
vllm serve /data/model/Qwen3.5-35B-A3B-W4A8 --trust-remote-code -tp 1 --max-num-seqs 64 --enable-chunked-prefill --max-model-len 16384 --host 0.0.0.0 --port 1234 --gpu-memory-utilization 0.90 --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "level": 0}'
Qwen3.5-35B-A3B
export VLLM_RPC_TIMEOUT=50000
export VLLM_ENFORCE_CUDA_GRAPH=1
export VLLM_KV_DISABLE_CROSS_GROUP_SHARE=1
vllm serve /data/model/Qwen3.5-35B-A3B --trust-remote-code -tp 4 --max-num-seqs 64 --enable-chunked-prefill --max-model-len 16384 --host 0.0.0.0 --port 1234 --gpu-memory-utilization 0.90 --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "level": 0}'
Qwen3.5-122B-A10B-W4A8
export VLLM_RPC_TIMEOUT=50000
export VLLM_ENFORCE_CUDA_GRAPH=1
export VLLM_W8A8_MOE_USE_W4A8=1
export VLLM_KV_DISABLE_CROSS_GROUP_SHARE=1
vllm serve /data/model/
Qwen3.5-122B-A10B-W4A8
--trust-remote-code --disable-cascade-attn --tensor-parallel-size 4 --gpu-memory-utilization 0.9 --enable_chunked_prefill --max-model-len 16384 --host 0.0.0.0 --port 1234 --max-num-seqs 64 --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "level": 0}'
Qwen3.5-122B-A10B
export VLLM_RPC_TIMEOUT=50000
export VLLM_ENFORCE_CUDA_GRAPH=1
export VLLM_KV_DISABLE_CROSS_GROUP_SHARE=1
vllm serve /data/model/Qwen3.5-122B-A10B --trust-remote-code --disable-cascade-attn --tensor-parallel-size 4 --gpu-memory-utilization 0.9 --pipeline-parallel-size 4 --enable_chunked_prefill --max-model-len 16384 --host 0.0.0.0 --port 1234 --max-num-seqs 64 --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "level": 0}'
Qwen3.5-397B-A17B-W4A8
export VLLM_RPC_TIMEOUT=50000
export VLLM_ENFORCE_CUDA_GRAPH=1
export VLLM_W8A8_MOE_USE_W4A8=1
export VLLM_KV_DISABLE_CROSS_GROUP_SHARE=1
vllm serve /data/model/Qwen3.5-397B-A17B-W4A8 --trust-remote-code --disable-cascade-attn --tensor-parallel-size 4 --gpu-memory-utilization 0.9 --pipeline-parallel-size 4 --enable_chunked_prefill --max-model-len 16384 --host 0.0.0.0 --port 1234 --max-num-seqs 64 --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "level": 0}'
客户端进行测试
再起一个窗口，不进入容器，输入下面命令
curl http://0.0.0.0:1234/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "/data/model/Qwen3.5-122B-A10B-W4A8",
"messages": [
{
"role": "user",
"content": "介绍一下湖南省常德市美食"
}
],
"temperature": 0.0,
"chat_template_kwargs": {
"enable_thinking": false
},
"max_tokens": 4096
}'
