今天承接上一篇,手写单词纠错
需要准备语料库- -
代码
'''贝叶斯要解决的问题是 逆概 问题'''
import re,collections
import pandas as pd
txt = ''
csv = 'frequency.csv'
# 获取文本中的英文单词 (作为先验概率的数据) 单词方案1 统计文章中出现的单词的频率
def words(test): return re.findall('[a-z]+', test.lower())
# def train(features):
# mode1 = collections.defaultdict(lambda :1)
# for f in features:
# mode1[f] += 1
# return mode1
# ---------------------------------------------------------------------------
# 单词方案2 加载词频文本
def train(csv):
model = collections.defaultdict(lambda :1)
data = pd.read_csv(csv,header=None,names=['words'],encoding='utf-8')
for w in range(1,len(data)+1):
model[data['words'][w-1]] = w
# print(model)
return model
#NWORDS = train(words(open(txt).read()))
NWORDS = train(csv)
alphabet = 'abcdefghijklmnopqrstuvwxyz'
# 编辑距离
# 返回所有与单词w编辑距离为1的集合
def edits1(word):
n = len(word)
return set(
[word[0:i] + word[i+1:] for i in range(n)] + # deletion
[word[0:i] + word[i+1] + word[i] + word[i+2:] for i in range(n-1)] + # transposition
[word[0:i] + c + word[i+1:] for i in range(n) for c in alphabet] + # alteration
[word[0:i] + c + word[i:] for i in range(n) for c in alphabet] # insetion
)
# 编辑距离为2
def edits2(word):
return set(e2 for e1 in edits1(word) for e2 in edits1(e1))
def know(words):
return set(w for w in words if w in NWORDS)
# 如果know(set) 非空, candidate 就会选择这个集合,而不会继续计算后面的
def correct(word):
candidate = know([word]) or know(edits1(word)) or know(edits2(word)) or [word]
return max(candidate, key=lambda w : NWORDS[w])
while True:
a = input("请输入一个单词:")
print('你可能想输入的单词是 :{}'.format(correct(a)))
print('-'*20)
需要语料库的下方发邮箱。。。
(来不及解释代码了,快上车。。。。又水一篇,下周解释)