python-Spider

爬取当当网排名前500的书籍信息

import re
import json
import requests


# 请求网页信息
def request_dang(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
    except requests.RequestException:
        return None


# 解析过滤
def parse_result(html):
    pattern = re.compile('<li>.*?list_num.*?(\d+).</div>.*?<img src="(.*?)".*?class="name".*?title="(.*?)">.*?class="star">.*?class="tuijian">(.*?)</span>.*?class="publisher_info">.*?target="_blank">(.*?)</a>.*?class="biaosheng">.*?<span>(.*?)</span></div>.*?<p><span\sclass="price_n">&yen;(.*?)</span>.*?</li>', re.S)
    items = re.findall(pattern, html)
    for item in items:
        yield {
            '排名': item[0],
            '图片地址': item[1],
            '书名': item[2],
            '推荐度': item[3],
            '作者': item[4],
            '出版时间': item[5],
            '价格': item[6]
        }


# 保存到文件
def write_to_file(item):
    print('开始写入数据 ====> ' + str(item))
    with open('Top500books.txt', 'a', encoding='UTF-8') as f:
        f.write(json.dumps(item, ensure_ascii=False) + '\n')
        f.close()


# 请求网页内容并写入文件
def main_dang(page):
    url = 'http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-recent30-0-0-1-' + str(page)
    html = request_dang(url)
    items = parse_result(html)
    for item in items:
        write_to_file(item)


if __name__ == '__main__':
    for i in range(1, 26):	# 按页循环
        main_dang(i)