SDP_LSTM 数据分析

# -*- codeing = utf-8 -*-
# @Time : 2021/8/17 11:13

import matplotlib.pyplot as plt
import numpy as np

pos_tag_vab=[]#词性标注字典
for line in open(file='D:\\pythonProject1\\CNN\\Relation-Classification-using-Bidirectional-LSTM-Tree-master\\data\\pos_tags.txt'):
    pos_tag_vab.append(line.strip())
print(pos_tag_vab)
depend_vab=[]#依存字典
for line in open(file='D:\\pythonProject1\\CNN\\Relation-Classification-using-Bidirectional-LSTM-Tree-master\\data\\dependency_types.txt'):
    depend_vab.append(line.strip())
relation_vab=[]#关系字典
for line in open(file='D:\\pythonProject1\\CNN\\Relation-Classification-using-Bidirectional-LSTM-Tree-master\\data\\relation_types.txt'):
    relation_vab.append(line.strip())
print(depend_vab,'\n',relation_vab)

import _pickle as pickle
import nltk
#加载语料资源
f = open('D:\\pythonProject1\\CNN\\Relation-Classification-using-Bidirectional-LSTM-Tree-master\\data\\train_pathsv3', 'rb')
words_seq, deps_seq, pos_tags_seq, word_path1, word_path2, dep_path1, dep_path2, pos_tags_path1, pos_tags_path2, pos_path1, pos_path2, childs_path1, childs_path2 = pickle.load(f)
# print(pos_tags_seq)
f.close()

relations = []
for line in open('D:\\pythonProject1\\CNN\\Relation-Classification-using-Bidirectional-LSTM-Tree-master\\data\\train_relationsv3.txt'):
    relations.append(line.strip().split()[1])
#频繁关系
most_commen_rels=nltk.FreqDist(relations).most_common( )
print(most_commen_rels)

#语料中的频繁词性
for i,pos in enumerate(pos_tags_seq):#pos_tag_seq由列表组成的
    if i==0:
        freq_pos=nltk.FreqDist(pos)
        print(freq_pos)
    else:
        freq_pos+=nltk.FreqDist(pos)
        # print(freq_pos)
most_commen_pos=freq_pos.most_common()
print(most_commen_pos)


for i,dep in enumerate(deps_seq):
    if i==0:
        freq_deps=nltk.FreqDist(dep)
    else:
        freq_deps+=nltk.FreqDist(dep)
# print(freq_deps)
most_commen_deps=freq_deps.most_common()
print(most_commen_deps)


deps=[]
freq_deps=[]
for i ,v in most_commen_deps[:25]:
    deps.append(i)
    freq_deps.append(v)
print(deps)
print(freq_deps)

indexes = np.arange(len(freq_deps))

# plt.bar(x=indexes,height=freq_deps,width=0.5,align='center',alpha=0.5,color='b')
# plt.xticks(indexes,deps,rotation='vertical')
width = 5
plt.title('依存句法')
plt.bar(indexes, freq_deps, align='center', alpha=0.5,color='g')
plt.xticks(indexes, deps, rotation='vertical')
plt.show()
print("Dependency Type :",depend_vab)



pos_tag=[]
freq_pos_tags=[]
for i,v in most_commen_pos[:25]:
    pos_tag.append(i)
    freq_pos_tags.append(v)

index=np.arange(len(pos_tag))
plt.bar(x=index,height=freq_pos_tags,width=0.8,align='center',color='g')
plt.xticks(index,pos_tag,rotation='vertical')
print('词性标注全部类型',pos_tag_vab)
plt.show()



rel_tag=[]
freq_rel_tag=[]
for i,v in most_commen_rels[:25]:
    rel_tag.append(i)
    freq_rel_tag.append(v)
index=np.arange(len(rel_tag))
plt.bar(index,height=freq_rel_tag,color='y')
plt.xticks(index,rel_tag,rotation='vertical')
print('关系类型',relation_vab)
plt.show()