通义千问-7B-Chat-8-9 离线整合包 使用 4bit 运行的代码;离线包自己去下载,把app.py 文件改成下面的,然后记得安装一个 bitsandbytes 的window 版本, 安装命令放在最后了
import os
#os.system('pip install tiktoken')
#os.system('pip install "modelscope" --upgrade -f https://pypi.org/project/modelscope/')
import gradio as gr
from modelscope.pipelines import pipeline
from modelscope.utils.constant import Tasks
from transformers import BitsAndBytesConfig
import torch
# os.environ['CUDA_VISIBLE_DEVICES'] = '1'
def clear_session():
return '', None
model_id = './qwen/Qwen-7B-Chat'
quantization_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type='nf4',
bnb_4bit_compute_dtype=torch.bfloat16)
pipe = pipeline(
task=Tasks.chat, model=model_id, device_map='auto', revision='