python爬虫 利用Beautifuloup爬取豆瓣影评

#coding=utf-8
from bs4 import BeautifulSoup
import re
import urllib2
import xlwt
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

#得到页面全部内容
def askURL(url):
    request = urllib2.Request(url)#发送请求
    try:
        response = urllib2.urlopen(request)#取得响应
        html= response.read()#获取网页内容
        #print html
    except urllib2.URLError, e:
        if hasattr(e,"code"):
            print e.code
        if hasattr(e,"reason"):
            print e.reason
    #print html
    return html

#获取相关内容
def getData(url):
    content = []
    for i in range(0,5):
        html=askURL(url+str(i*10))
        soup=BeautifulSoup(html,'lxml')
        lists=soup.find_all('div',class_="main review-item")
        #print lists
        for li in lists:
            item=[]
            title=li.h2.a.string
            item.append(title)
            #print title
            auther=li.find('a',property="v:reviewer").string
            item.append(str(auther))
            #print auther
            #<a class="subject-img" href="https://movie.douban.com/subject/1482072/"> <img alt="穿普拉达的女王" rel="v:image" src="https://img3.doubanio.com/view/photo/s_ratio_poster/public/p735379215.jpg" title="穿普拉达的女王"/> </a>
            movie_title=li.find('a',class_="subject-img").img['title']
            item.append(str(movie_title))
            #print movie_title
            #<a class="subject-img" href="https://movie.douban.com/subject/1482072/">
            movie_link=li.find('a',class_="subject-img")['href']
            item.append(str(movie_link))
            #print movie_link
            #<span class="allstar50 main-title-rating" property="v:rating" title="力荐"></span>
            star=li.find('span',class_=re.compile('allstar[\d]+ main-title-rating'))['title']   #具有不同的数字，所以得正则表达式匹配一下
            item.append(str(star))
            #print star
            #<a class="reply" href="https://movie.douban.com/review/9342401/#comments">0回应</a>
            reply_num=li.find('a',class_="reply").string
            reply_num=str(reply_num)[:-6]  #去掉后面的回应两个字
            item.append(reply_num)
            #print reply_num
            #<h2><a href="https://movie.douban.com/review/9342401/">吃胖了也是很可爱的女孩，艾米丽。</a></h2>
            review_link=li.h2.a['href']
            #print review_link
            item.append(str(review_link))
            #< div class ="short-content" >
            html_review=askURL(review_link)
            review_soup=BeautifulSoup(html_review,'lxml')
            review_content=review_soup.find('div',class_="review-content clearfix").children   #求标签的所有的孩子的P标签，用children方法
            all_review =" "
            for r_c in review_content:
                all_review = all_review.replace('<p>','')  #去除p标签
                all_review = all_review.replace('</p>','')
                all_review=all_review+str(r_c)
            item.append(all_review)
            #<span id="r-useful_count-9342401">26</span>
            #有用数
            useful_count=li.find('span',id=re.compile('r-useful_count-[\d]+')).string.strip()  #去除空格
            if len(useful_count)!=0:
                useful_count=useful_count
            else:
                useful_count=0
            item.append(str(useful_count))

            #没用数
            useless_count = li.find('span', id=re.compile('r-useless_count-[\d]+')).string.strip()  # 去除空格
            if len(useless_count) != 0:
                useless_count = useless_count
            else:
                useless_count = 0
            item.append(str(useless_count))


            content.append(item)
    #print content
    return content


#将相关数据写入excel中
def saveData(datalist,savepath):
    book=xlwt.Workbook(encoding='utf-8',style_compression=0)
    sheet=book.add_sheet('豆瓣最受欢迎影评',cell_overwrite_ok=True)
    col=('标题','作者','影片名','影片详情链接','推荐级','回应数','影评链接','影评','有用数','没用数')
    for i in range(0,10):
        sheet.write(0,i,col[i])#列名
    for i in range(0,50):#总共50条影评
        data=datalist[i]
        for j in range(0,10):
            sheet.write(i+1,j,data[j])#数据
    book.save(savepath)#保存

def main():
    url='http://movie.douban.com/review/best/?start='
    datalist=getData(url)
    savapath=u'豆瓣100条影评.xlsx'
    saveData(datalist,savapath)

main()

爬取结果如图：