From 49a2dfaf90982079239d116fcfac9ca72a8fc2b5 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Tue, 2 Apr 2024 22:45:20 +0800 Subject: [PATCH] update vllm example --- README.md | 5 +++-- README_zh.md | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1b218418..778d90b7 100644 --- a/README.md +++ b/README.md @@ -369,10 +369,11 @@ Use `python src/train_bash.py -h` to display arguments description. ### Deploy with OpenAI-style API and vLLM ```bash -CUDA_VISIBLE_DEVICES=0 API_PORT=8000 python src/api_demo.py \ +CUDA_VISIBLE_DEVICES=0,1 API_PORT=8000 python src/api_demo.py \ --model_name_or_path mistralai/Mistral-7B-Instruct-v0.2 \ --template mistral \ - --infer_backend vllm + --infer_backend vllm \ + --vllm_enforce_eager ``` ### Use ModelScope Hub diff --git a/README_zh.md b/README_zh.md index a8390f21..7b02c55d 100644 --- a/README_zh.md +++ b/README_zh.md @@ -369,10 +369,11 @@ docker compose -f ./docker-compose.yml up -d ### 使用 OpenAI 风格 API 和 vLLM 部署 ```bash -CUDA_VISIBLE_DEVICES=0 API_PORT=8000 python src/api_demo.py \ +CUDA_VISIBLE_DEVICES=0,1 API_PORT=8000 python src/api_demo.py \ --model_name_or_path mistralai/Mistral-7B-Instruct-v0.2 \ --template mistral \ - --infer_backend vllm + --infer_backend vllm \ + --vllm_enforce_eager ``` ### 使用魔搭社区