50行代码爬取Top500图书导入TXT文档

50行代码爬取Top500图书导入TXT文档

import re   #正则表达式，进行文字提取
import requests
import json

def main(page):
    #声明爬取网址
    baseurl = "http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-recent30-0-0-1-" + str(page)
    #爬取网页内容
    datalist = getData(baseurl)
    #保存网页数据
    savepath = "Top500_book.txt"
    saveData(datalist,savepath)

#得到数据
def getData(baseurl):
    html = askURL(baseurl)
    datalist = parse_result(html)
    return datalist
#对源码进行解析
def parse_result(html):
   pattern = re.compile('<li>.*?list_num.*?(\d+).</div>.*?<img src="(.*?)".*?class="name".*?title="(.*?)">.*?class="star">.*?class="tuijian">(.*?)</span>.*?class="publisher_info">.*?target="_blank">(.*?)</a>.*?class="biaosheng">.*?<span>(.*?)</span></div>.*?<p><span\sclass="price_n">&yen;(.*?)</span>.*?</li>',re.S)
   items = re.findall(pattern,html)
   for item in items:
       yield {
           'range': item[0],
           'iamge': item[1],
           'title': item[2],
           'recommend': item[3],
           'author': item[4],
           'times': item[5],
           'price': item[6]
       }
#获取网页源码
def askURL(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
    except requests.RequestException:
        return None
#保存数据到txt文本文档
def saveData(datalst,savepath):
    print("save....")
    for item in datalst:
        with open(savepath, 'a', encoding='UTF-8') as f:
            f.write(json.dumps(item, ensure_ascii=False) + '\n')
            f.close()

if __name__ == '__main__':
    #for循环实现翻页
    for i in range(1,26):
        main(i)

【运行结果】