爬虫

我的第一个爬虫

import requests
from lxml import etree
from time import sleep
import json

def get_response(url):
    response = requests.get(url).text
    return response

#将response转为HTML
def handel_html(response):
    html = etree.HTML(response)
    return html


def save_to_file(filename,data):
    with open(filename,'w') as f:
        data = json.dump(data, fp=f)
        f.write(str(data))
        print('保存完成在',filename)

#使用xpath解析网页
def handel_page_data(response):
    #保存该页面的所有图书信息
    page_book = []
    #每个图书信息都保存在class= "indent"的div标签下的table标签内
    book_table = handel_html(response).xpath('//div[@class="indent"]//table')
    for book_td in book_table:
        #保存单个图书信息
        book = { }
        # 图书名保存在class="pl2"的div的第一个a标签内的"title"属性内
        book['name'] = book_td.xpath('.//div[@class="pl2"]//a[1]/@title')
        # 其他信息保存在class="pl"的div的第一个a标签内的"title"属性内
        other_info = book_td.xpath('.//p[@class="pl"]/text()')
        other_info = other_info[0].split('/')
        book['data'] = other_info[-2]
        book['price'] = other_info[-1]
        book['press'] = other_info[-3]
        book['author'] =other_info[:-3]
        book['author'] = ' '.join(book['author'])
        #放入改业所有图书信息
        page_book.append(book)
    return page_book
def get_next_url(response):
    #下一页信息保存在text() = '后页>'中的a标签的herf属性
    html = handel_html(response)
    next_url = html.xpath('//span[@class="next"]//a/@href')
    if len(next_url)>0:
        return next_url[0]
    else:
        return  None

def run():
    #保存图书信息
    books_info = []
    url = 'https://book.douban.com/top250?icn=index-book250-all'
    #获取网页响应
    response = get_response(url)
    html = handel_html(response)
    #解析网页内容
    page_books = handel_page_data(response)
    print(page_books)
    #放入所有图书信息中
    books_info.extend(page_books)
    #获取下一页Url
    next_url = get_next_url(response)
    print(next_url)
    #当下一夜存在时
    while next_url:
        #设置延时防治封ip
        sleep(2)
        response = get_response(next_url)
        html = handel_html(response)
        # 解析网页内容
        page_books = handel_page_data(response)
        print(page_books)
        books_info.extend(page_books)
        # 获取下一页Url
        next_url = get_next_url(response)
    for i in books_info:
        print(i)
    print('数据获取完成，一共%d条'%len(books_info))
    print('正在保存数据')
    save_to_file(filename='data.json',data=books_info)
if __name__ == '__main__':
    run()