初探nlp 词频统计，去停顿词

一些简单且实用的python操作

rainbow = open(r'C:\Users\Linsinan\Desktop\彩虹.txt')
text = rainbow.read().split()

text.count('the') # 词频计算
words = set(text) # 不重复的元组数据
len(words) # 有19657个不重复的单词
'draw'[::-1] # 对一个单词进行反转

{w for w in words if w == w[::-1] and len(w) > 4} # 长度大于4且反转后还是自己的单词
{w for w in words if w == w[::-1] and len(w) > 4} # set + lambda的操作，学到了

简单的词频统计

import string
from urllib.request import urlopen
import nltk
import matplotlib.pyplot as plt
from nltk.corpus import stopwords

shakespeare = urlopen('http://composingprograms.com/shakespeare.txt')
text = shakespeare.read().decode().lower().split()
words = set(text)
countsDict = {index: text.count(index) for index in words}
# countsDict = nltk.FreqDist(text)，最快最直接的方式

# 去掉标点
for i in string.punctuation:
    try:
        countsDict.pop(i)
    except:
        pass


# 前10名频率的单词
rankWord = sorted(countsDict, key=lambda x: countsDict[x], reverse=True)
values = [countsDict[i] for i in rankWord[:10]]
plt.bar(range(len(values)), values, tick_label=rankWord[:10])
plt.show()


# 前10名的非停用词
for i in stopwords.words('english'):
    try:
        countsDict.pop(i)
    except:
        pass

rankWord = sorted(countsDict, key=lambda x: countsDict[x], reverse=True)
values = [countsDict[i] for i in rankWord[:10]]
plt.bar(range(len(values)), values, tick_label=rankWord[:10])
plt.show()