下载模型
apt-get install git-lfs 或者yum install git-lfs
git clone https://www.modelscope.cn/Qwen/Qwen2-VL-7B-Instruct.git
基于base4.2镜像启动容器
docker run --shm-size="64g" -it -v /usr/src:/usr/src -v /lib/modules:/lib/modules -v /dev:/dev -v /home:/home --name=vltest --network=host --privileged --cap-add=ALL --pid=host corex:4.2.0
进入容器后执行下面操作
pip3 uninstall opencv-python3
pip3 install transformers==4.47.1 -i https://pypi.tuna.tsinghua.edu.cn/simple
启动vllm api
export ENABLE_FLASH_ATTENTION_WITH_HEAD_DIM_PADDING=1
python3 -m vllm.entrypoints.openai.api_server \
--served-model-name Qwen2-VL-7B-Instruct \
--model /home/deepseek/Qwen2-VL-7B-Instruct \
--trust-remote-code
测试
curl http://localhost:8000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "Qwen2-VL-7B-Instruct",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "描述这张图片"}
,
{"type": "image_url", "image_url": {"url": "https://img-s-msn-com.akamaized.net/tenant/amp/entityid/AAOEcgc.img"}}
]
}
]
}'
