python 统计单词后面的搭配_Python nltk：查找没有点分隔单词的搭配

您可以使用WordPunctTokenizer将标点符号与单词分开,然后使用apply_word_filter()过滤掉带有标点符号的双字母组合.

同样的事情可以用于三元组,因为没有找到句子边界上的搭配.

from nltk import bigrams

from nltk import collocations

from nltk import FreqDist

from nltk.collocations import *

from nltk import WordPunctTokenizer

content_part = """test. foo 0 test. foo 1 test.

foo 2 test. foo 3 test. foo 4 test, foo 4 test."""

tokens = WordPunctTokenizer().tokenize(content_part)

bigram_measures = collocations.BigramAssocMeasures()

word_fd = FreqDist(tokens)

bigram_fd = FreqDist(bigrams(tokens))

finder = BigramCollocationFinder(word_fd, bigram_fd)

finder.apply_word_filter(lambda w: w in ('.', ','))

scored = finder.score_ngrams(bigram_measures.raw_freq)

print tokens

print sorted(finder.nbest(bigram_measures.raw_freq,2),reverse=True)

输出：

['test', '.', 'foo', '0', 'test', '.', 'foo', '1', 'test', '.', 'foo', '2', 'test', '.', 'foo', '3', 'test', '.', 'foo', '4', 'test', ',', 'foo', '4', 'test', '.']

[('4', 'test'), ('foo', '4')]