环境:
本地部署的LLM:Qwen2-72B-Instruct-GPTQ-Int4
使用vllm启动提供服务,并且采用兼容OPENAI接口服务
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
from langchain_community.chat_message_histories import ChatMessageHistory
openai_api_key = 'token-abc123'
openai_api_base = 'http://172.18.25.21:8000/v1'
mode_id = 'Qwen2-72B-Instruct-GPTQ-Int4'
chat = ChatOpenAI(api_key=openai_api_key, base_url=openai_api_base, model=mode_id)
prompt = ChatPromptTemplate.from_messages(
[
('system', '你是一个全能性助手,尽你的最大能力回答'),
MessagesPlaceholder(variable_name='messages'),
]
)
chain = prompt | chat
message_history = ChatMessageHistory()
while True:
user_input = input('人:')
if user_input == 'exit':
break
message_history.add_user_message(user_input)
result = chain.invoke({
'messages': message_history.messages,
})
ai_ret = result.content
print(f'AI: {ai_ret}')
message_history.add_ai_message(ai_ret)