命名实体识别(BiLSTM+CRF)

最新推荐文章于 2024-08-05 10:13:02 发布

咸鱼肝

最新推荐文章于 2024-08-05 10:13:02 发布

阅读量799

点赞数 3

文章标签： lstm nlp nec

本文链接：https://blog.csdn.net/m0_53272995/article/details/121022022

版权

数据来源
- 获取
数据标注
- 效果
模型搭建
代码部分

数据来源

百度百科：一战的主要战争人物介绍。

获取

import requests
import re
import pandas as pd
from lxml import html
from lxml import etree

url = 'https://baike.baidu.com/item/%E7%AC%AC%E4%B8%80%E6%AC%A1%E4%B8%96%E7%95%8C%E5%A4%A7%E6%88%98/68516?fr=aladdin'
headers = {
   
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36'
}
res = requests.get(url=url,headers=headers)
s = res.text
html = etree.HTML(s)
p = html.xpath('//div[@class="para"]/text()')
p = [i for i in p if len(i)>20]
data = pd.DataFrame()
data['文字'] = p
data.to_csv('一战信息.txt',index=False)

数据标注

在线标注平台

效果

在这里插入图片描述
以txt格式导出

模型搭建

import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import tqdm
from tensorflow.keras.models import Sequential
from tensorflow.keras.callbacks import TensorBoard
from tensorflow.keras.optimizers import Adam
from tensorflow_addons.layers import CRF

class MyBiLSTMCRF:
    def __init__(self, vocabSize, maxLen, tagIndexDict,tagSum,sequenceLengths=None,vecSize=100,learning_rate=0.01):
        self.vocabSize = vocabSize
        self.vecSize = vecSize
        self.maxLen = maxLen
        self.tagSum = tagSum
        self.sequenceLengths=sequenceLengths
        self.tagIndexDict=tagIndexDict
        self.learning_rate=learning_rate

        self.buildBiLSTMCRF()

    def getTransParam(self,y,tagIndexDict):
        self.trainY=np.argmax(y,axis=-1)
        yList=self.trainY.tolist()
        transParam=np.zeros([len(list(tagIndexDict.keys())),len(list(tagIndexDict.keys()))])
        for rowI in range(len(yList)):
            for colI in range(len(yList[rowI])-1):
                transParam[yList[rowI][colI]][yList[rowI][colI+1]]+=1
        for rowI in range(transParam.shape[0]):
            transParam[rowI]=transParam[rowI]/np.sum(transParam[rowI])
        return transParam
    
    def buildBiLSTMCRF(self):

        myModel=Sequential()
        myModel.add(tf.keras.layers.Input(shape=(self.maxLen,)))
        myModel.add(tf.keras.layers.Embedding(self.vocabSize, self.vecSize))
        myModel.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(
                    self.tagSum, return_sequences=True, activation="tanh"), merge_mode='sum'))
        myModel.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(
                    self.tagSum, return_sequences=True, activation="softmax"), merge_mode='sum'))
        crf=CRF(self.tagSum,name=