回答下列问题：
（１）怎样才能识别出语言数据中明显用于分类的特征？
（２）怎样才能构建用于自动执行语言处理任务的语言模型？
（３）从这些模型中我们可以学到哪些关于语言的知识？
决策树，朴素贝叶斯分类器和最大熵（shang）分类
一 监督式分类
＃性别鉴定
创建分类
def gender_features(word):
return {'last_letter': word[-1]}
gender_features('Shrek')
{'last_letter': 'k'}
from nltk.corpus import names
import random
names = ([(name, 'male') for name in names.words('male.txt')] +
[(name, 'female') for name in names.words('female.txt')])
random.shuffle(names)
import nltk
featuresets = [ (gender_features(n), g) for (n,g) in names ]
train_set, test_set = featuresets[500:], featuresets[:500] ＃训练集和测试集
classifier = nltk.NaiveBayesClassifier.train(train_set)
classifier.classify(gender_features('Neo'))
'male'
classifier.classify(gender_features('Trinity'))
'female'
print nltk.classify.accuracy(classifier, test_set) #评估
0.75
classifier.show_most_informative_features(5) #哪些特征对于区分名字的性别是最有效的
Most Informative Features
last_letter = u'a' female : male = 33.4 : 1.0
last_letter = u'k' male : female = 30.8 : 1.0
last_letter = u'f' male : female = 17.3 : 1.0
last_letter = u'p' male : female = 10.5 : 1.0
last_letter = u'd' male : female = 10.0 : 1.0＃选择正确的特征
def gender_features2(name):
features = {}
features["firstletter"] = name[0].lower()
features["lastletter"] = name[-1].lower()
for letter in 'abcdefghijklmnopqrstuvwxyz':
features["count(%s)" % letter] = name.lower().count(letter)
features["has(%s)" % letter] = (letter in name.lower())
return features
gender_features2('JJohn')
featuresets = [(gender_features2(n), g) for (n,g) in names]
train_set, test_set = featuresets[500:], featuresets[:500]
classifier = nltk.NaiveBayesClassifier.train(train_set) #使用朴素贝叶斯分类器
print nltk.classify.accuracy(classifier, test_set)
0.776#一种能有效完善特征集的方法称为错误分析。首先，选择开发集，其中包含用于创建模型的语料数据。然后将这种开发集分为训练集和开发测试集
train_names = names[1500:]
devtest_names = names[500:1500]
test_names = names[:500]
train_set = [(gender_features(n), g) for (n,g) in train_names]
devtest_set = [(gender_features(n),g) for (n,g) in devtest_names]
test_set = [(gender_features(n),g) for (n,g) in test_names]
classifier = nltk.NaiveBayesClassifier.train(train_set)
print nltk.classify.accuracy(classifier, devtest_set)
0.766errors = []
for (name, tag) in devtest_names:
guess = classifier.classify(gender_features(name))
if guess != tag:
errors.append( (tag, guess, name) )
for (tag, guess, name) in sorted(errors):
print 'correct=%-8s guess=%-8s name=%-30s' % (tag, guess, name)
correct=female guess=male name=Abagael
correct=female guess=male name=Adel
correct=female guess=male name=Alys
correct=female guess=male name=Amargo
correct=female guess=male name=Ambur... #调整特征提取器使其包含两个字母后缀的特征
def gender_features(word):
return {'suffix1': word[-1:],
'suffix2': word[-2:]}
train_set = [(gender_features(n), g) for (n,g) in train_names]
devtest_set = [(gender_features(n),g) for (n,g) in devtest_names]
classifier = nltk.NaiveBayesClassifier.train(train_set)
print nltk.classify.accuracy(classifier, devtest_set)0.784 ＃文档分类
将电影评论语料库归类为正面或负面
from nltk.corpus import movie_reviews
documents = [(list(movie_reviews.words(fileid)), category)
for category in movie_reviews.categories()
for fileid in movie_reviews.fileids(category)]
random.shuffle(documents)#文档分类的特征提取器，其特征表示每个词是否在一个给定的文档中
all_words = nltk.FreqDist(w.lower() for w in movie_reviews.words())
word_features = all_words.keys()[:2000]
def document_features(document):
document_words = set(document)
features = {}
for word in word_features:
features['contains(%s)' % word] = (word in document_words)
return features
print document_features(movie_reviews.words('pos/cv957_8737.txt'))
{u'contains(corporate)': False, u'contains(barred)': False, u'contains(batmans)': False, u'contains(menacing)': False, u'contains(rags)': False, u'contains(inquires)': False, #训练和测试分类器以进行文档分类
featuresets = [(document_features(d),c) for (d,c) in documents]
train_set, test_set = featuresets[100:], featuresets[:100]
classifier = nltk.NaiveBayesClassifier.train(train_set)
print nltk.classify.accuracy(classifier, test_set)
0.73classifier.show_most_informative_features(5) #找出哪些特征是分类器发现的并且是最有信息量的
Most Informative Features
contains(sans) = True neg : pos = 9.1 : 1.0
contains(mediocrity) = True neg : pos = 7.8 : 1.0
contains(dismissed) = True pos : neg = 6.9 : 1.0
contains(testament) = True pos : neg = 6.5 : 1.0
contains(bruckheimer) = True neg : pos = 6.4 : 1.0＃词性标注
from nltk.corpus import brown
suffix_fdist = nltk.FreqDist()
for word in brown.words():
word = word.lower()
suffix_fdist[word[-1:]] += 1
suffix_fdist[word[-2:]] += 1
suffix_fdist[word[-3:]] += 1
from operator import itemgetter
common_suffixes = sorted(suffix_fdist.items(), key=itemgetter(1), reverse=True)
common_suffixes[:100]
[(u'e', 202946),
(u',', 175002),
(u'.', 152999),
(u's', 128722),
(u'd', 105687),
(u't', 94459),common_suf = [ suffix[0] for suffix in common_suffixes][:100]
common_suf
def pos_features(word): features = {} for suffix in common_suf: features['endswith(%s)'%suffix] = word.lower().endswith(suffix) return featurestagged_words = brown.tagged_words(categories='news') tagged_words[0] (u'The', u'AT') len(tagged_words) 100554 len(pos_features(tagged_words[0][0])) 100 pos_features(tagged_words[0][0]) {u"endswith('')": False, u"endswith(')": False, u"endswith('s)": False, u'endswith(()': False, u'endswith())': False, u'endswith(,)': False, featuresets = [(pos_features(n),g) for (n,g) in tagged_words] size = int(len(featuresets) * 0.1) size Out[52]: 10055 train_set,test_set = featuresets[size:], featuresets[:size]classifier = nltk.DecisionTreeClassifier.train(train_set) #决策树 nltk.classify.accuracy(classifier, test_set) 0.6270512182993535 classifier.classify(pos_features('cats')) Out[54]: u'NNS' #决策树的优点是容易解释，甚至可以它们以伪代码形式输出 print classifier.pseudocode(depth=4) if endswith(the) == False: if endswith(,) == False: if endswith(s) == False: if endswith(.) == False: return u'.' if endswith(.) == True: return u'.' if endswith(s) == True: if endswith(is) == False: return u'PP$' if endswith(is) == True: return u'BEZ' if endswith(,) == True: return u',' if endswith(the) == True: return u'AT'＃探索上下文语境
不是只传递已标注的词，而是传递整个（未标注的）句子，以及目标词的索引
＃特征检测器
>>>def pos_features(sentence, i): features = {"suffix(1)": sentence[i][-1:], "suffix(2)": sentence[i][-2:], "suffix(3)": sentence[i][-3:]} if i == 0: features["prev-word"] = "<START>" else: features["prev-word"] = sentence[i-1] return features brown.sents()[0][7] Out[62]: u'an' brown.sents()[0][8] Out[63]: u'investigation' pos_features(brown.sents()[0], 8) ###### 四个特征 {'prev-word': u'an', 'suffix(1)': u'n', 'suffix(2)': u'on', 'suffix(3)': u'ion'}tagged_sents = brown.tagged_sents(categories='news') featuresets = [] for tagged_sent in tagged_sents: untagged_sent = nltk.tag.untag(tagged_sent) for i, (word, tag) in enumerate(tagged_sent): featuresets.append( (pos_features(untagged_sent, i), tag) ) size = int(len(featuresets) * 0.1) 10055train_set, test_set = featuresets[size:], featuresets[:size] classifier = nltk.NaiveBayesClassifier.train(train_set) nltk.classify.accuracy(classifier, test_set) 0.7891596220785678＃序列分类
在词性标注的例子中，可以使用各种不同的序列分类器模型为给定的句子中的所有词选择词性标注
一种称为连续分类或贪婪序列分类的序列分类器策略，为第一个输入找到最有可能的类标签，然后在此基础上找到下一个输入的最佳的标签。这个过程可以不断重复直到所有的输入都被贴上标签。
特征提取器
def pos_features(sentence, i, history): features = {"suffix(1)": sentence[i][-1:], "suffix(2)": sentence[i][-2:], "suffix(3)": sentence[i][-3:] } if i == 0: features["prev-word"] = "<START>" features["prev-tag"] = "<START>" else: features["prev-word"] = sentence[i-1] features["pre-tag"] = history[i-1] return featuresclass ConsecutivePosTagger(nltk.TaggerI): def __init__(self, train_sents): train_set = [] for tagged_sent in train_sents: untagged_sent = nltk.tag.untag(tagged_sent) history = [] for i, (word, tag) in enumerate(tagged_sent): featureset = pos_features(untagged_sent, i, history) train_set.append((featureset, tag)) history.append(tag) ###### self.classifier = nltk.NaiveBayesClassifier.train(train_set) def tag(self, sentence): history = [] for i, word in enumerate(sentence): featureset = pos_features(sentence, i, history) tag = self.classifier.classify(featureset) history.append(tag) return zip(sentence, history)tagged_sents = brown.tagged_sents(categories='news') size = int(len(tagged_sents) * 0.1) train_sents, test_sents = tagged_sents[size:], tagged_sents[:size] tagger = ConsecutivePosTagger(train_sents) print tagger.evaluate(test_sents)＃其他序列分类方法
这种方法的缺点是一旦做出决定便无法更改。例如：如果决定将一个词标注为名词，但后来发现应该是动词，那也没有办法修复我们的错误了。解决这个问题的方法是采取转型策略。转型联合分类的工作原理是为输入的标签创建一个初始值，然后反复提炼该值，尝试修复相关输入之间的不一致
另一种方案是为词性标记所有可能的序列打分，选择总得分最高的序列。隐马尔科夫模型就采取了这种方法。隐ｍａｅｒｋｅｆｕｍｏｘ类似于连续分类器，不光考虑输入也考虑已预测标记的历史。然而，不是简单地找出一个给定词的单个最好标签，而是为标记产生一个概率分布。然后这些概率结合起来计算标记序列的概率得分，最后选择最高概率的标记序列。不过，可能的标签序列数量相当大。给定拥有３０个标签的标记集，大约有600万亿(30^10)中方式来标记一个１０个词的句子。为了避免单独考虑所有这些可能的序列，隐马尔科夫模型要求特征提取器只考虑最近的标记（或最近的n个标记，其中n是相当小的）。由于这种限制，它可以使用动态规划来有效地找出最有可能的标记序列。特别是，对每个连续的词索引i，当前的及以前的每个可能的标记都将计算得分。这种基础的方法被两个更先进的模型所采用，它们被称为最大熵马尔科夫模型和线性链条件随机场模型；但为标记序列打分用的是不同的算法。
二 监督式分类的举例
＃句子分割
第一步是获得一些已被分割成句子的数据，将它转换成一种适合提取特征的形式
sents = nltk.corpus.treebank_raw.sents() tokens = [] boundaries = set() offset = 0 for sent in nltk.corpus.treebank_raw.sents(): tokens.extend(sent) offset += len(sent) boundaries.add(offset - 1)def punct_features(tokens, i): return { 'next-word-capitailized': tokens[i+1][0].isupper(), 'prevword': tokens[i-1].lower(), 'punct': tokens[i], 'prev-word-is-one-char': len(tokens[i-1]) == 1}featuresets = [(punct_features(tokens, i), (i in boundaries)) for i in range(1, len(tokens)-1) if tokens[i] in '.?!'] size = int(len(featuresets) * 0.1) train_set, test_set = featuresets[size:], featuresets[:size] classifier = nltk.NaiveBayesClassifier.train(train_set) nltk.classify.accuracy(classifier, test_set) 0.936026936026936def segment_sentences(words): #基于分类的断句器 start = 0 sents = [] for i, word in words: if word in '.?!' and classifier.classify(punct_features(words, i)) == True: sents.append(words[start:i+1]) start = i+1 if start < len(words): sents.append(words[start:]) return sents＃识别对话行为类型
表述行为的陈述句，问候，问题，回答，断言和说明都可以被认为是基于语言的行为类型。识别对话中隐含言语下的对话行为是理解谈话的重要步骤。
利用NPS聊天语料库建立一个分类器，用来识别新的即时消息帖子的对话行为类型。
posts = nltk.corpus.nps_chat.xml_posts()[:10000] #每个帖子的XML注释def dialogue_act_features(post): #特征提取器 features = {} for word in nltk.word_tokenize(post): features['contains(%s)' % word.lower()] = True return features featuresets = [(dialogue_act_features(post.text), post.get('class')) for post in posts] ({'contains(gay)': True, 'contains(im)': True, 'contains(left)': True, 'contains(name)': True, 'contains(now)': True, 'contains(this)': True, 'contains(with)': True}, 'Statement') #陈述句size = int(len(featuresets) * 0.1) #分类器 train_set, test_set = featuresets[size:], featuresets[:size] classifier = nltk.NaiveBayesClassifier.train(train_set) print nltk.classify.accuracy(classifier, test_set) 0.668＃识别文字蕴涵
(Recognizing textual entailment, RTE)是判断文本Ｔ内的一个给定片段是否继承另一个叫做“假设”的文本。迄今为止，已经有４个RTE挑战赛，在那里共享的开发和测试数据会提供给参赛队伍。
def rte_features(rtepair): extractor = nltk.RTEFeatureExtractor(rtepair) features = {} features['word_overlap'] = len(extractor.overlap('word')) features['word_hyp_extra'] = len(extractor.hyp_extra('word')) features['ne_overlap'] = len(extractor.overlap('ne')) features['ne_hyp_extra'] = len(extractor.hyp_extra('ne')) return featuresrtepair = nltk.corpus.rte.pairs(['rte3_dev.xml'])[33] extractor = nltk.RTEFeatureExtractor(rtepair) print extractor.text_words set(['Organisation', 'Shanghai', 'Asia', 'four', 'at', 'operation', 'SCO', 'Iran', 'Soviet', 'Davudi', 'fight', 'China', 'association', 'fledgling', 'was', 'that', 'republics', 'former', 'Co', 'representing', 'Russia', 'Parviz', 'central', 'meeting', 'together', 'binds', 'terrorism.'])print extractor.hyp_words set(['member', 'SCO.', 'China']) print extractor.overlap('word') set([]) print extractor.overlap('ne') set(['China']) print extractor.hyp_extra('word') set(['member'])＃扩展到大型数据集
纯Python的分类不是很快，建议探索NLTK与外部机器学习包的接口技术，
三 评估
测试集
准确度
精确度和召回率
混淆矩阵
交叉验证
四 决策树
熵和信息增益
五 朴素贝叶斯分类器
潜在概率模型
零计数和平滑
非二元特征
独立的朴素性
双重计数的原因
六 最大熵分类器
最大熵模型
熵的最大化
生成式分类器对比条件分类器
七 为语言模式建模
模型告诉我们什么？
八 深入阅读
使用Weka, Mallet, TADM 和 MegaM
扫一扫
1123
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
