贝叶斯实现拼写检查器

贝叶斯实现拼写检查器

import re,collections
def words(text):return re.findall('[a-z]+',text.lower())
def train(features):
    model=collections.defaultdict(lambda:1)
    for f in features:
        model[f]+=1
    return model
NWORDS=train(words(open('big.txt').read()))
alphabet='abcdefghijklmnopqrstuvwxyz'
def edits1(word):
    n=len(word)
    return set([word[0:i]+word[i+1:] for i in range(n)]+
               [word[0:i]+word[i+1]+word[i]+word[i+2:] for i in range(n-1)]+
               [word[0:i]+c+word[i+1:] for i in range(n) for c in alphabet]+
               [word[0:i]+c+word[i:] for i in range(n+1) for c in alphabet])
def known_edist2(word):
    return set(e2 for e1 in edist1(word) for e2 in edist1(e1) if e2 in edist1(e1) if e2 in NWORDS)
def known(words): return set(w for w in words if w in NWORDS)
def correct(word):
    candidates=known([word]) or known(edits1(word)) or known_edist2(word) or [word]
    return max(candidates,key=lambda w:NWORDS[w])
#appl #appla #learw #tess #morw
print(correct('norw'))

求解：

argmaxc P(c|w)->argmaxc P(w|c)P(c)/P(w)

argmaxc：用来枚举所有可能的c并选取最大的

P(c):先验概率，键入c的可能性是多大

P(w|c):想键入c敲成w

#把语料中的单词全部抽取出来，转成小写，并且除去单词中间的符号
def words(text):return re.findall('[a-z]+',text.lower())
def train(features):
    model=collections.defaultdict(lambda:1)
    for f in features:
        model[f]+=1
    return model
NWORDS=train(words(open('big.txt').read()))
print(NWORDS)

编辑距离：

两个词之间的编辑距离，定义为使用了几次插入，删除，交换，替换的操作从一个词变到另一个词

#返回所有与单词w编辑距离为1的集合
def edits1(word):
    n=len(word)
    return set([word[0:i]+word[i+1:] for i in range(n)]+#deletion
               [word[0:i]+word[i+1]+word[i]+word[i+2:] for i in range(n-1)]+#transposition
               [word[0:i]+c+word[i+1:] for i in range(n) for c in alphabet]+#alteration
               [word[0:i]+c+word[i:] for i in range(n+1) for c in alphabet])#insertion
#返回所有与单词w编辑距离为2的集合
#在这些距离小于2的词中间，只把那些正确的单词作为候选词
def known_edist2(word):
    return set(e2 for e1 in edist1(word) for e2 in edist1(e1) if e2 in edist1(e1) if e2 in NWORDS)