参考链接:
https://blog.csdn.net/lcwdzl/article/details/78493637
https://blog.csdn.net/liu_zhlai/article/details/52125174?spm=1001.2014.3001.5501
代码源码地址:
https://github.com/lankuohsing/Study_NLP
1. 机械分词算法
1.1. 正向最大匹配算法
custom_dict={"南京","南京市","市长","长江","大桥","江大桥",}
input_sentence="南京市长江大桥"
max_word_len=0
for word in custom_dict:
if len(word)>max_word_len:
max_word_len=len(word)
if len(input_sentence)<max_word_len:
max_word_len=len(input_sentence)
start=0
seg_results=[]
while start<len(input_sentence):
temp_len=max_word_len
if len(input_sentence)-start<max_word_len:
temp_len=len(input_sentence)-start
while temp_len>0:
sub_sentence=input_sentence[start:start+temp_len]
if sub_sentence in custom_dict:
seg_results.append(sub_sentence)
start+=temp_len
break
else:
temp_len-=1
# 没有子串匹配,则单独成词
if temp_len==0:
seg_results.append(input_sentence[start:start+1])
start+=1
print(seg_results)
1.2. 逆向最大匹配算法
custom_dict={"南京","南京市","市长","长江","大桥","江大桥"}
input_sentence="南京市长江大桥"
max_word_len=0
for word in custom_dict:
if len(word)>max_word_len:
max_word_len=len(word)
if len(input_sentence)<max_word_len:
max_word_len=len(input_sentence)
end=len(input_sentence)
seg_results=[]
while end>0:
temp_len=max_word_len
if end<max_word_len:
temp_len=end
while temp_len>0:
sub_sentence=input_sentence[end-temp_len:end]
if sub_sentence in custom_dict:
seg_results.append(sub_sentence)
end-=temp_len
break
else:
temp_len-=1
# 没有子串匹配,则单独成词
if temp_len==0:
sub_sentence=input_sentence[end-1:end]
seg_results.append(sub_sentence)
end-=1
print(seg_results)