2.最大正向匹配法
这是一个基于词表的分词方法。
主要思想是:把一个句子从左向右扫描一遍,
- 遇到词典中的词就标识出来
- 遇到复合词找最长的词匹配
- 遇到不认识的字串就分割为单字词
词表我们已经建立成功了(中文信息处理之最大正向匹配法(上)),接着我们开始根据上述三点写主体程序,代码如下。
def fenci(s,maxlen,Fenci_list):
# temp 为辅助变量
temp = s
result = ''
while temp != '':
lens = maxlen
if len(temp) < lens:
lens = len(temp)
word = temp[:lens]
while word not in Fenci_list:
word = word[:len(word)-1]
if len(word) == 1:
break
result = result+ word + '/'
temp = temp[len(word):]
return result
3.评价程序
这里我们使用三个指标:
由于文本比较大,程序进行分词的时间比较长,所以我截取了一部分进行测试。
步骤:
- 读取测试文本,使用程序进行分词,将结果保存在一个文件中。
- 读取检验结果,由于没有现成的结果,从文本中抽取原本的分词结果。
- 计算 R、P、F
代码如下:
with open(filepath,'r') as f:
result = f.read()
# 系统分割方案
result = fenci(text,maxlen=5,Fenci_list=Fenci_list)
with open(resultpath,'w') as f:
f.write(result)
result1 = [x for x in result.split('/') if x]
pattern = re.compile(r'\s*(.*?)/[a-z]+')
with open(targetpath,'r') as f:
targetfile = f.read()
target = re.findall(pattern,targetfile)
#print(target)
# same 表示两个数组相同元素个数
same = [l for l in result1 if l in target]
samelength = len(same)
resultlength = len(result1)
targetlength = len(target)
# R P F 计算
R = samelength/targetlength
P = samelength/resultlength
F = 2*P*R/(P+R)
print('samelength:',samelength)
print('targetlength:',targetlength)
print('resultlegth:',resultlength)
print("召回率:",R)
print("准确率:",P)
print("F 测度:",F)
效果图:
总结
第一次做中文信息处理实验,基本理解了最大正向匹配法。对于最大正向匹配法的 maxlen 变量,我没有进行处理,直接有用户进行输入,这一点还有待改进。python 的使用也不是很熟练,代码结构有待优化。
附上所有代码
# fenci.py
# 分词
import re
from collections import Counter
import operator
from functools import reduce
# 初始文件和结果文件的位置
filepath = '材料/199801.txt'
resultpath = '材料/result.txt'
def creatFenci():
with open(filepath,'r') as f:
text = f.read()
# pattern1 匹配文本信息,条件是两个空格和/之间的
pattern1 = re.compile(r' (.*?)/')
pattern3 = re.compile(r'\[(.*?)]')
pattern4 = re.compile(r'[a-z\[\]\s\/]*')
# 除去文本中 [] 符号,方便抽取颗粒比较细的中文词汇
text1 = re.sub(r'[\[\]]*','',text)
fenci_list = re.findall(pattern1,text1)
buchongci = re.findall(pattern3,text)
buResult = []
# 从 buchongci 中抽取中文
# eg'中国/ns 驻/v 加拿大/ns 使馆/n 教育处/n'变成'中国驻加拿大使馆教育处'
for i in buchongci:
heci = re.sub(pattern4,'',i)
buResult.append(heci)
Fenci_list = buResult + fenci_list
print(len(Fenci_list),len(fenci_list),len(buResult))
# 统计分词词频写入字典并进行排序
data = dict(Counter(Fenci_list))
data2 = sorted(data.items(),key = lambda e:e[1],reverse=False)
# 词表写入文件,以元组的格式
# with open(resultpath,'w') as f:
# for i in data2:
# f.write(str(i)+'\n')
return Fenci_list
def fenci(s,maxlen,Fenci_list):
# temp1 为辅助变量
temp1 = s
temp2 = ''
while temp1 != '':
lens = maxlen
if len(temp1) < lens:
lens = len(temp1)
word = temp1[:lens]
while word not in Fenci_list:
word = word[:len(word)-1]
if len(word) == 1:
break
temp2 = temp2+ word + '/'
temp1 = temp1[len(word):]
return temp2
if __name__ == '__main__':
s = input("输入句子:")
maxlen = int(input("输入最大词长:"))
print(fenci(s,maxlen))
# fenciTest.py
from fenci import fenci
import re
filepath = '材料/测试文本.txt'
resultpath = '材料/fenciResult.txt'
targetpath = '材料/测试结果.txt'
with open('材料/Fenci.txt','r') as f:
Fenci_list = f.readlines()
# print(Fenci_list)
with open(resultpath,'r') as f:
result = f.read()
# 系统分割方案
# with open(filepath,'r') as f:
# text = f.read()
# result = fenci(text,maxlen=5,Fenci_list=Fenci_list)
# with open(resultpath,'w') as f:
# f.write(result)
# print('fenci over')
result1 = [x for x in result.split('/') if x]
#print(result1)
# 目标分割方法
pattern = re.compile(r'\s*(.*?)/[a-z]+')
with open(targetpath,'r') as f:
targetfile = f.read()
target = re.findall(pattern,targetfile)
#print(target)
# same 表示两个数组相同元素个数
same = [l for l in result1 if l in target]
samelength = len(same)
resultlength = len(result1)
targetlength = len(target)
# R P F 计算
R = samelength/targetlength
P = samelength/resultlength
F = 2*P*R/(P+R)
print('samelength:',samelength)
print('targetlength:',targetlength)
print('resultlegth:',resultlength)
print("召回率:",R)
print("准确率:",P)
print("F 测度:",F)