python3 案例分享--爬虫实战--爬小说

2019独角兽企业重金招聘Python工程师标准>>>

用python3,爬起点小说网站，生成txt小说（小试牛刀）,废话不多说，直接上代码：

import requests
from lxml import etree
import os

# 设计模式 -- 面向对象
class Spider(object):
    def start_request(self):
        # 1. 请求一级页面拿到HTML源代码，抽取小说名、小说链接 创建文件夹
        response = requests.get("https://www.qidian.com/all")
        html = etree.HTML(response.text)
        Bigtit_list = html.xpath('//div[@class="book-mid-info"]/h4/a/text()')
        Bigsrc_list = html.xpath('//div[@class="book-mid-info"]/h4/a/@href')
        for Bigtit, Bigsrc in zip(Bigtit_list, Bigsrc_list):
            if os.path.exists(Bigtit) == False:
                os.mkdir(Bigtit)
            self.next_file(Bigtit, Bigsrc)

    def next_file(self, Bigtit, Bigsrc):
        # 2. 请求二级页面拿到HTML源代码，抽取章名、章链接
        response = requests.get("https:" + Bigsrc)
        html = etree.HTML(response.text)
        Littit_list = html.xpath('//ul[@class="cf"]/li/a/text()')
        Litsrc_list = html.xpath('//ul[@class="cf"]/li/a/@href')
        for Littit, Litsrc in zip(Littit_list, Litsrc_list):
            self.finally_file(Littit, Litsrc, Bigtit)

    def finally_file(self, Littit, Litsrc, Bigtit):
        # 3. 请求三级页面拿到HTML源代码，抽取文章内容，保存数据
        response = requests.get("https:" + Litsrc)
        html = etree.HTML(response.text)
        content = "\n".join(html.xpath('//div[@class="read-content j_readContent"]/p/text()'))
        file_name = Bigtit + "\\" + Littit + ".txt"
        print("正在保存小说文件：" + file_name)
        with open(file_name, "w", encoding="utf-8") as f:
            f.write(content)


spider = Spider()
spider.start_request()

生成的txt小说路径可以自定义路径哈。代码里默认是与源码同级目录 。

 

转载于:https://my.oschina.net/lyleluo/blog/3049372