lmdeploy部分
准备环境
拉取环境
/root/share/install_conda_env_internlm_base.sh lmdeploy
激活环境
conda activate lmdeploy
下载模型
cp -r /root/share/model_repos/internlm2-chat-7b /root/model/
安装
# 解决 ModuleNotFoundError: No module named 'packaging' 问题
pip install packaging
# 使用 flash_attn 的预编译包解决安装过慢问题
pip install /root/share/wheels/flash_attn-2.4.2+cu118torch2.0cxx11abiTRUE-cp310-cp310-linux_x86_64.whl
pip install 'lmdeploy[all]==v0.2.1'
转换internLM2-Chat-7B为将模型转为 lmdeploy TurboMind 的格式
lmdeploy convert internlm2-chat-7b /root/model/internlm2-chat-7b
启动 TurboMind推理+API服务
lmdeploy serve api_server ./workspace \
--server_name 0.0.0.0 \
--server_port 23333 \
--instance_num 64 \
--tp 1
Opencompass
安装环境
conda create --name opencompass --clone=/root/share/conda_envs/internlm-base
conda activate opencompass
git clone https://github.com/open-compass/opencompass
cd opencompass
pip install -e .
数据准备
# 解压评测数据集到 data/ 处
cp /share/temp/datasets/OpenCompassData-core-20231110.zip /root/opencompass/
unzip OpenCompassData-core-20231110.zip
config文件准备,文件名:eval_internlm2_chat_7b_turbomind_api.py
from mmengine.config import read_base
from opencompass.models.turbomind_api import TurboMindAPIModel
with read_base():
from .datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
meta_template = dict(
round=[
dict(role='HUMAN', begin='<|im_start|>user\n', end='<|im_end|>\n'),
dict(role='BOT', begin='<|im_start|>assistant\n', end='<|im_end|>\n', generate=True),
],
eos_token_id=92542)
models = [
dict(
type=TurboMindAPIModel,
abbr='internlm2-chat-7b-turbomind',
path="/root/model/internlm2-chat-7b",
api_addr='http://0.0.0.0:23333',
max_out_len=100,
max_seq_len=2048,
batch_size=16,
meta_template=meta_template,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
推理
python run.py configs/eval_internlm2_chat_7b_turbomind_api.py -w outputs/turbomind/internlm2-chat-7b