【自然语言处理课程作业】作业三命名实体识别与关系提取

Q1使用CCL2021中文命名实体识别数据集，完成命名实体识别任务。

# ! unzip data.zip

#1. 读取数据
char_vocab_path = "./data/char_vocabs.txt" # 字典文件
train_data_path = "./data/train_data.txt" # 训练测试数据 

special_words = ['<PAD>', '<UNK>'] # 特殊词表示


# CCL2021数据标签：
label2idx = {"O": 0,
             "B-Symptom": 1, 
             "I-Symptom": 2,
             "B-Drug_Category": 3, 
             "I-Drug_Category": 4,
             "B-Drug": 5, "I-Drug": 6,
             "B-Medical_Examination":7, 
             "I-Medical_Examination":8,
             "B-Operation":9, 
             "I-Operation":10
             }

# 索引和BIO标签对应
idx2label = {idx: label for label, idx in label2idx.items()}

# 读取字符词典文件
with open(char_vocab_path, "r", encoding="utf8") as fo:
    char_vocabs = [line.strip() for line in fo]
char_vocabs = special_words + char_vocabs

# 字符和索引编号对应
idx2vocab = {idx: char for idx, char in enumerate(char_vocabs)}
vocab2idx = {char: idx for idx, char in idx2vocab.items()}

#2. 读取训练语料，将数据集划分为训练集和测试集
# 读取训练语料
def read_corpus(corpus_path, vocab2idx, label2idx, flag):
    datas, labels = [], []
    with open(corpus_path, encoding='utf-8') as fr:
        lines = fr.readlines()
    sent_, tag_ = [], []
    # print(len(lines))
    if flag=="train":
        lines=lines[:int(len(lines)*0.7)]
    else:
        lines=lines[int(len(lines)*0.7):]
    print(len(lines))
    for line in lines:
        if line != '\n':
            [char, label] = line.strip().split()
            sent_.append(char)
            tag_.append(label)
        else:
            sent_ids = [vocab2idx[char] if char in vocab2idx else vocab2idx['<UNK>'] for char in sent_]
            tag_ids = [label2idx[label] if label in label2idx else 0 for label in tag_]
            datas.append(sent_ids)
            labels.append(tag_ids)
            sent_, tag_ = [], []
    return datas, labels

# 加载训练集 7成
train_datas, train_labels = read_corpus(train_data_path, vocab2idx, label2idx, flag="train")
# 加载测试集 3成
test_datas, test_labels = read_corpus(train_data_path, vocab2idx, label2idx, flag="test")

print(train_datas[8])
print([idx2vocab[idx] for idx in train_datas[8]])
print(train_labels[8])
print([idx2label[idx] for idx in train_labels[8]])

#下载项目demo
!pip install git+https://gitee.com/myyy1997/keras-contrib.git

#3. 构建BiLSTM+CRF模型，对数据进行训练

import numpy as np
import keras
from keras.models import Sequential
from keras.models import Model
from keras.layers import Masking, Embedding, Bidirectional, LSTM, Dense, Input, TimeDistributed, Activation
from keras.preprocessing import sequence
from keras_contrib.layers import CRF
from keras_contrib.losses import crf_loss
from keras_contrib.metrics import crf_viterbi_accuracy
from keras import backend as K
K.clear_session()

# 参数
EPOCHS = 2
BATCH_SIZE = 64
EMBED_DIM = 128
HIDDEN_SIZE = 6