Python Ajax爬虫

爬B站，Ajax，翻页

'''
Created on 2015-10-9
'''
#encoding=utf-8
from lxml import html
from time import sleep
import requests

#xpath
title_xpath = "//div[@class='l-item']/a[@class='title']/text()"
play_xpath = "//div[@class='w-info']/i[@class='gk']/text()"
comment_xpath = "//div[@class='w-info']/i[@class='dm']/text()"

#information
titles = []
plays = []
comments = []

headers = {'X-Requested-With': ' XMLHttpRequest'}
i = 1;
while len(titles) < 50:
    page = requests.get('http://www.bilibili.com/list/hot-27-%s-2015-10-05~2015-10-12.html' % i, headers=headers)
    tree = html.fromstring(page.text)

    titles += tree.xpath(title_xpath)
    plays += tree.xpath(play_xpath)
    comments += tree.xpath(comment_xpath)
    print "We got %s titles , %s number of playing and %s number of comment" % (len(titles), len(plays), len(comments))
    
    i = i + 1
    sleep(3)

i = 0
with open("information.txt","wb") as out:
    while i < len(titles) and i < len(plays) and i < len(comments):
        out.write(titles[i].encode('utf-8'))
        out.write('\t'.encode('utf-8'))
        out.write(plays[i].encode('utf-8'))
        out.write('\t'.encode('utf-8'))
        out.write(comments[i].encode('utf-8'))
        out.write('\r\n'.encode('utf-8'))
        i += 1
print "WRITE DONE"
print "Please open the txt to see the information"
print "The window will close after 3 seconds"
sleep(3)