我的第一个爬虫
import requests
from lxml import etree
from time import sleep
import json
def get_response(url):
response = requests.get(url).text
return response
#将response转为HTML
def handel_html(response):
html = etree.HTML(response)
return html
def save_to_file(filename,data):
with open(filename,'w') as f:
data = json.dump(data, fp=f)
f.write(str(data))
print('保存完成在',filename)
#使用xpath解析网页
def handel_page_data(response):
#保存该页面的所有图书信息
page_book = []
#每个图书信息都保存在class= "indent"的div标签下的table标签内
book_table = handel_html(response).xpath('//div[@class="indent"]//table')
for book_td in book_table:
#保存单个图书信息
book = { }
# 图书名保存在class="pl2"的div的第一个a标签内的"title"属性内
book['name'] = book_td.xpath('.//div[@class="pl2"]//a[1]/@title')
# 其他信息保存在class="pl"的div的第一个a标签内的"title"属性内
other_info = book_td.xpath('.//p[@class="pl"]/text()')
other_info = other_info[0].split('/')
book['data'] = other_info[-2]
book['price'] = other_info[-1]
book['press'] = other_info[-3]
book['author'] =other_info[:-3]
book['author'] = ' '.join(book['author'])
#放入改业所有图书信息
page_book.append(book)
return page_book
def get_next_url(response):
#下一页信息保存在text() = '后页>'中的a标签的herf属性
html = handel_html(response)
next_url = html.xpath('//span[@class="next"]//a/@href')
if len(next_url)>0:
return next_url[0]
else:
return None
def run():
#保存图书信息
books_info = []
url = 'https://book.douban.com/top250?icn=index-book250-all'
#获取网页响应
response = get_response(url)
html = handel_html(response)
#解析网页内容
page_books = handel_page_data(response)
print(page_books)
#放入所有图书信息中
books_info.extend(page_books)
#获取下一页Url
next_url = get_next_url(response)
print(next_url)
#当下一夜存在时
while next_url:
#设置延时防治封ip
sleep(2)
response = get_response(next_url)
html = handel_html(response)
# 解析网页内容
page_books = handel_page_data(response)
print(page_books)
books_info.extend(page_books)
# 获取下一页Url
next_url = get_next_url(response)
for i in books_info:
print(i)
print('数据获取完成,一共%d条'%len(books_info))
print('正在保存数据')
save_to_file(filename='data.json',data=books_info)
if __name__ == '__main__':
run()