基本概念
序列标注
序列标注可以认为是token级别的文本分类。
常见序列标注任务
NER
NER(Named-entity recognition) :分辨出文本中的名词和实体是 person人名, organization组织机构名, location地点名还是其他。
POS
POS (Part-of-speech tagging): 对token进行词性标注
Chunk
将同一个短语的tokens组块放在一起
一个NER任务
需要安装的库
- datasets transformers seqeval
数据加载
from datasets import load_dataset, load_metric
datasets = load_dataset("conll2003")
label_list = datasets["train"].features[f"{task}_tags"].feature.names # 查看训练集分类标签
标签:
- ‘O’ for no special entity
- ‘PER’ for person
- ‘ORG’ for organization
- ‘LOC’ for location
- ‘MISC’ for miscellaneous
每一种实体类别又分别有B-(实体开始的token)前缀和I-(实体中间的token)前缀
数据预处理
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
import transformers
assert isinstance(tokenizer, transformers.PreTrainedTokenizerFast)
#tokenize
def tokenize_and_align_labels(examples):
tokenized_inputs = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)
labels = []
for i, label in enumerate(examples[f"{task}_tags"]):
word_ids = tokenized_inputs.word_ids(batch_index=i)
previous_word_idx = None
label_ids = []
for word_idx in word_ids:
# Special tokens have a word id that is None. We set the label to -100 so they are automatically
# ignored in the loss function.
if word_idx is None:
label_ids.append(-100) #特殊字符的label设置为-100,在模型中会被忽略掉不计算loss
# We set the label for the first token of each word.
elif word_idx != previous_word_idx:
label_ids.append(label[word_idx])
# For the other tokens in a word, we set the label to either the current label or -100, depending on
# the label_all_tokens flag.
else:
label_ids.append(label[word_idx] if label_all_tokens else -100)
previous_word_idx = word_idx
labels.append(label_ids)
tokenized_inputs["labels"] = labels
return tokenized_inputs
tokenize_and_align_labels(datasets['train'][:5])
tokenized_datasets = datasets.map(tokenize_and_align_labels, batched=True)
微调预训练模型
from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer
task = "ner"
model_checkpoint = "distilbert-base-uncased"
batch_size = 16
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint, num_labels=len(label_list))
args = TrainingArguments(
f"test-{task}",
evaluation_strategy = "epoch",
learning_rate=2e-5,
per_device_train_batch_size=batch_size,
per_device_eval_batch_size=batch_size,
num_train_epochs=3,
weight_decay=0.01,
)
#加载数据收集器
from transformers import DataCollatorForTokenClassification
data_collator = DataCollatorForTokenClassification(tokenizer)
#评价指标
import numpy as np
metric = load_metric("seqeval")
def compute_metrics(p):
predictions, labels = p
predictions = np.argmax(predictions, axis=2)
# Remove ignored index (special tokens)
true_predictions = [
[label_list[p] for (p, l) in zip(prediction, label) if l != -100]
for prediction, label in zip(predictions, labels)
]
true_labels = [
[label_list[l] for (p, l) in zip(prediction, label) if l != -100]
for prediction, label in zip(predictions, labels)
]
results = metric.compute(predictions=true_predictions, references=true_labels)
return {
"precision": results["overall_precision"],
"recall": results["overall_recall"],
"f1": results["overall_f1"],
"accuracy": results["overall_accuracy"],
}
# trainer
trainer = Trainer(
model,
args,
train_dataset=tokenized_datasets["train"],
eval_dataset=tokenized_datasets["validation"],
data_collator=data_collator,
tokenizer=tokenizer,
compute_metrics=compute_metrics
)
# 模型训练
trainer.train()
# 模型评价
trainer.evaluate()