from sentence_transformers import SentenceTransformer, SentencesDataset
from sentence_transformers import InputExample, evaluation, losses
from torch.utils.data import DataLoader
import os
import random
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
model = SentenceTransformer('../pre_model/distiluse-base-multilingual-cased')
# 加载数据集
#文件内容格式: ('气性坏疽', '产气荚膜杆菌感染', 1.0)
def load_data(filename):
datas = []
with open(filename, encoding='utf-8') as f:
for l in f:
l = l.strip()
ls = eval(l)
if ls is None:
continue
datas.append(ls)
random.shuffle(datas)
return datas
datas = load_data('../data/train_data3.txt')
print('train_datas len:',len(datas))
eval_datas = load_data('../data/eval_data3.txt')
print('eval_datas len:',len(eval_datas))
# 构造训练数据
train_da
使用sentence-transformer进行fine-tune
最新推荐文章于 2024-04-24 17:22:02 发布
![](https://img-home.csdnimg.cn/images/20240711042549.png)