Python2.7 爬虫实践：豆瓣电影影评分析

reference from ：hang

segmentfault.com/a/1190000010473819

本人先看到以上，觉得挺好玩，所以就跟着原作者的思路在撸一遍代码

后来发现了几个问题

本人的python 2.7 原作者为python3

所以其中的一些funtion/属性 也不支持

所以做了一下修改

安装必要的lib 之后会有文章进行介绍

好了 就先上代码吧

__author__ = 'Helen Huang'

# encoding=utf8
import sys
import warnings
warnings.filterwarnings("ignore")
reload(sys)
sys.setdefaultencoding('utf8')
from urllib2 import urlopen
import bs4
import time
import re
import jieba
import numpy
import codecs
import matplotlib.pyplot as plt
import matplotlib
from wordcloud import WordCloud
import pandas as pd


matplotlib.rcParams['figure.figsize'] = (10.0, 5.0)

def getMovieId(title):
    resp = urlopen('https://movie.douban.com/nowplaying/hangzhou/')
    html_data = resp.read().decode('utf-8')
    soup = bs4.BeautifulSoup(html_data,"html.parser")
    content = soup.find_all('div',id='nowplaying')
    movieStrList=content[0].find_all('li',class_ ='list-item')
    movieList=[]
    for item in movieStrList:
        movie_dit={}
        movie_dit['id']=item['data-subject']
        movie_dit['name']=item['data-title']
        movieList.append(movie_dit)
        item['data-title']=item['data-title'].strip('')
        if unicode(item['data-title']) == (title):
            movieId =item['data-subject']
            if unicode(item['data-title']) == (title):
                return movieId

def getCommentsById(movieId,pageNum):

  #避免uncode出现在云图，所以我直接将结果保存为str
    for i in range(10):
        num = i + 1
        if pageNum >0 :
            start =(num-1) * 20
        else:
            return False
        reqUrl='https://movie.douban.com/subject/' + movieId + '/comments' +'?' +'start=' + str(start) + '&limit=20'
        print reqUrl
        resp = urlopen(reqUrl)
        html = resp.read().decode('utf-8')
        #print html
        soup = bs4.BeautifulSoup(html,"html.parser")
        comContent = soup.find_all('div',id='comments')
        #print comContent
        commentStr=comContent[0].find_all('div',class_ ='comment')
        commentList =[]
        comments=''
        for comment in commentStr:
            c = comment.find_all('p')[0].string
            if c is not None:
                commentList.append(c)
                comments =comments+str(c).strip().strip('\n')
                print c
        return comments
if __name__ == '__main__':
    print 'start ....'
    title=u'杀破狼·贪狼'
    movieId = getMovieId(title)
    print 'movie id is:'
    print movieId
    comments = getCommentsById(movieId,10)
    comments=comments.replace(' ','')
    print comments
    #使用正则表达式去除标点符号
    pattern = re.compile(r'[\u4e00-\u9fa5]+')
    filterdata = re.findall(pattern, comments)
    cleaned_comments = ''.join(filterdata)
    cleaned_comments= comments
    #使用结巴分词进行中文分词
    segment = jieba.lcut(cleaned_comments)
    words_df=pd.DataFrame({'segment':segment})

    #去掉停用词 #如果有电影领域的停用词就更好了
    #stopwords=pd.read_csv('D:\python\stopwords.txt',index_col=False,sep="\t",names=['stopword'], encoding='utf-8')#quoting=3全不引用

    #python27 没有quoting 属性
    stopwords=pd.read_csv('D:\python\stopwords_copy.txt',names=['stopword'], encoding='utf-8')
    words_df=words_df[~words_df.segment.isin(stopwords.stopword)]
    #统计词频
    words_stat=words_df.groupby(by=['segment'])['segment'].agg({"计数":numpy.size})
    words_stat=words_stat.reset_index().sort_index(by=["计数"],ascending=False)
    print words_stat.head() 

    #用词云进行显示
  
    word_frequence = {x[0]:x[1] for x in words_stat.head(100).values}

  wordcloud=WordCloud(font_path="D:\python\sourcehansansi.ttf",background_color='white',max_font_size=80).generate_from_frequencies(word_frequence)

#这里没有用fit_word
    plt.figure()
    plt.imshow(wordcloud)
    plt.axis("off")
    #plt.savefig()
    plt.show()

    print 'end ...'

#我发现了的问题 

#表情词没有过滤

#如果一句话里面，用户重复了多遍的话，词频就有影响了

#stopwords 应该更加领域化

#wordcloud还可以设置多种样式

#以下是我的结果图