英文文本预处理
总结非常好的blog
Lemmatisation和Stemming(NLTK pos_tag word_tokenize)
#方法一
from nltk.stem import WordNetLemmatizer
def lemmatize_all(sentence): #Input:sentence
wnl=WordNetLemmatizer()
for word,tag in pos_tag(word_tokenize(sentence)): #分词+词性标注
if tag.startswith('NN'):
yield wnl.lemmatize(word,pos='n')
elif tag.startswith('VB'):
yield wnl.lemmatize(word, pos='v')
elif tag.startswith('JJ'):
yield wnl.lemmatize(word, pos='a')
elif tag.startswith('R'):
yield wnl.lemmatize(word, pos='r')
else:
yield word
train_f=[]
for i in range(len(train_feature)): #train_feature:sentence就行
train_f.append(''.join(lemmatize_all(train_feature[i])))
#方法二
python 新语法get
yield的用法详解——最简单,最清晰的解释
def foo(num):
while num<10:
num+=1
yield num
for n in foo(0):
print(n)
#_________________________________________________#
分词
新功能get
TreebankWordTokenizer依据PennTreebank语料库的约定,通过分离缩略词来实现切分
PunktWordTokenizer通过分离标点来实现切分的,每一个单词都会被保留