贝叶斯实现拼写检查器
import re,collections
def words(text):return re.findall('[a-z]+',text.lower())
def train(features):
model=collections.defaultdict(lambda:1)
for f in features:
model[f]+=1
return model
NWORDS=train(words(open('big.txt').read()))
alphabet='abcdefghijklmnopqrstuvwxyz'
def edits1(word):
n=len(word)
return set([word[0:i]+word[i+1:] for i in range(n)]+
[word[0:i]+word[i+1]+word[i]+word[i+2:] for i in range(n-1)]+
[word[0:i]+c+word[i+1:] for i in range(n) for c in alphabet]+
[word[0:i]+c+word[i:] for i in range(n+1) for c in alphabet])
def known_edist2(word):
return set(e2 for e1 in edist1(word) for e2 in edist1(e1) if e2 in edist1(e1) if e2 in NWORDS)
def known(words): return set(w for w in words if w in NWORDS)
def correct(word):
candidates=known([word]) or known(edits1(word)) or known_edist2(word) or [word]
return max(candidates,key=lambda w:NWORDS[w])
#appl #appla #learw #tess #morw
print(correct('norw'))
求解:
argmaxc P(c|w)->argmaxc P(w|c)P(c)/P(w)
argmaxc:用来枚举所有可能的c并选取最大的
P(c):先验概率,键入c的可能性是多大
P(w|c):想键入c敲成w
#把语料中的单词全部抽取出来,转成小写,并且除去单词中间的符号
def words(text):return re.findall('[a-z]+',text.lower())
def train(features):
model=collections.defaultdict(lambda:1)
for f in features:
model[f]+=1
return model
NWORDS=train(words(open('big.txt').read()))
print(NWORDS)
编辑距离:
两个词之间的编辑距离,定义为使用了几次插入,删除,交换,替换的操作从一个词变到另一个词
#返回所有与单词w编辑距离为1的集合
def edits1(word):
n=len(word)
return set([word[0:i]+word[i+1:] for i in range(n)]+#deletion
[word[0:i]+word[i+1]+word[i]+word[i+2:] for i in range(n-1)]+#transposition
[word[0:i]+c+word[i+1:] for i in range(n) for c in alphabet]+#alteration
[word[0:i]+c+word[i:] for i in range(n+1) for c in alphabet])#insertion
#返回所有与单词w编辑距离为2的集合
#在这些距离小于2的词中间,只把那些正确的单词作为候选词
def known_edist2(word):
return set(e2 for e1 in edist1(word) for e2 in edist1(e1) if e2 in edist1(e1) if e2 in NWORDS)