'''This following code will set the CURL_CA_BUNDLE environment variable to an empty string in the Python os module'''
import os
os.environ['CURL_CA_BUNDLE'] = ''
from datasets import load_dataset
from transformers import AutoTokenizer
from transformers import DataCollatorWithPadding
import evaluate
import numpy as np
from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer
def preprocess_function(examples):
return tokenizer(examples["text"], truncation=True)
# 训练集
imdb = load_dataset("imdb")
# 原始格式化
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased", mirror="tuna")
# 提取训练集中的text列
tokenized_imdb = imdb.map(preprocess_function, batched=True)
#
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
print(data_collator)
# 评估
accuracy = evaluate.load("accuracy")
# 评估准确性
def compute_metrics(eval_pred):
predictions, labels = eval_pred
predictions = np.argmax(predictions, axis=1)
return accuracy.compute(predictions=predictions, references=labels)
id2label = {0: "NEGATIVE", 1: "POSITIVE"}
label2id = {"NEGATIVE": 0, "POSITIVE": 1}
model = AutoModelForSequenceClassification.from_pretrained(
"distilbert-base-uncased", num_labels=2, id2label=id2label, label2id=label2id
)
# 训练参数
training_args = TrainingArguments(
output_dir="my_awesome_model",
learning_rate=2e-5,
per_device_train_batch_size=16,
per_device_eval_batch_size=16,
num_train_epochs=2,
weight_decay=0.01,
evaluation_strategy="epoch",
save_strategy="epoch",
load_best_model_at_end=True,
push_to_hub=False,
)
# 用训练参数创建出训练类
trainer = Trainer(
model=model,
args=training_args,
train_dataset=tokenized_imdb["train"],
eval_dataset=tokenized_imdb["test"],
tokenizer=tokenizer,
data_collator=data_collator,
compute_metrics=compute_metrics,
)
trainer.train()
运行结果:
推测测试
'''This following code will set the CURL_CA_BUNDLE environment variable to an empty string in the Python os module'''
import torch
import os
os.environ['CURL_CA_BUNDLE'] = ''
text = "i want kiss you"
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("./my_awesome_model/checkpoint-3126/")
inputs = tokenizer(text, return_tensors="pt")
from transformers import AutoModelForSequenceClassification
model = AutoModelForSequenceClassification.from_pretrained("./my_awesome_model/checkpoint-3126/")
with torch.no_grad():
logits = model(**inputs).logits
predicted_class_id = logits.argmax().item()
model.config.id2label[predicted_class_id]
会根据你的text的感情色彩给出倾向
POSITIVE
NEGATIVE
总结,这只是一个例子,准确率并不能说很高,只是一个学习的开始。