爬虫实战之爬虫小说

爬取静态网页之爬虫小说

---

复现爬虫小说

这里不做理论介绍，可以移步这里爬虫理论学习或者自己搜。

目标URL：笔趣阁

引入库：

import sys
import requests
from bs4 import BeautifulSoup

看看小说的正文

url = 'https://www.bqkan8.com/1_1094/5403177.html'  # 目标url
response = requests.get(url=url)  # 获取网页HTML
html = response.text  # .text 返回的是Beautifulsoup根据猜测的编码方式将content内容编码成字符串。
bf = BeautifulSoup(html, features="lxml")  # 不写features="lxml"，会warning：BeautifulSoup库未写明解析器警告
texts = bf.find_all('div', class_='showtxt')  # 在bf中寻找标签为div，class为showtxt的语句
texts = texts[0].text.replace('\xa0'*8, '\n\n')  # 替换
print(texts)

代码结果：

小说所有目录

# 目录
server = 'https://www.bqkan8.com/'
url = 'https://www.bqkan8.com/1_1094/'
response = requests.get(url)
response.encoding = response.apparent_encoding  # 解决返回字符串乱码问题

html = response.text
bshtml = BeautifulSoup(html, features="lxml")
bshtml = bshtml.find_all('div', class_='listmain')  

a_bf = BeautifulSoup(str(bshtml), features="lxml")
a = a_bf.find_all('a')

for each in a[13:]:
    print(each.string, server + each.get('href'))

代码结果：

最终组合代码：

class downloader(object):
    def __init__(self):
        self.server = 'https://www.bqkan8.com/'
        self.url = 'https://www.bqkan8.com/1_1094/'
        self.names = []  # 存放章节名
        self.urls = []  # 存放章节链接
        self.nums = 0  # 章节数


    def get_download_url(self):
        req = requests.get(url=self.url)
        html = req.text
        bf = BeautifulSoup(html, features="lxml")
        div = bf.find_all('div', class_='listmain')
        a_bf = BeautifulSoup(str(div), features="lxml")
        a = a_bf.find_all('a')
        self.nums = len(a[13:])
        for each in a[13:]:
            self.names.append(each.string)
            self.urls.append(self.server+each.get('href'))
        print(self.urls)
        return self.urls


    def get_contents(self, url):
        req = requests.get(url=url)
        html = req.text
        bf = BeautifulSoup(html, features="lxml")
        texts = bf.find_all('div', class_='showtxt')
        texts = texts[0].text.replace('\xa0'*8, '\n\n')
        # print(texts)
        return texts


    def wrist(self, name, path, text):
        with open(path, 'a', encoding='utf-8') as f:
            f.write(name+'\n')
            f.writelines(text)
            f.write('\n\n')


if __name__ == "__main__":
    dl = downloader()
    dl.get_download_url()
#    dl.get_contents(dl.get_download_url())
    print('《一年永恒》开始下载：')
    for i in range(dl.nums):
        dl.wrist(dl.names[i], '一念永恒.txt', dl.get_contents(dl.urls[i]))
        print("已完成第{}章小说爬取".format(i))
    print('《一年永恒》下载完成')

代码结果:

参考博客：复现的文章出处，非常值得一看

自己的爬虫小说

目标网站：纵横中文网
目标小说：剑来

思路：

1. 拿到所有章节名和章节链接
2. 根据章节链接找到每章小说正文
3. 根据章节名分类保存小说

# 引入库
import requests
from bs4 import BeautifulSoup

看看小说正文

# 每章小说正文，以第一章为例：
url = 'http://book.zongheng.com/chapter/672340/36898237.html'  # 第一章小说正文的url
req = requests.get(url=url)
html = req.text
bf = BeautifulSoup(html, features="lxml")
texts = bf.find_all('div', class_='content')
texts = texts[0].text.replace('</p><p>', '\n\n')  # 把</p><p>替换成换行
print(texts)

F12检查后发现，正文在

里：

效果图：

小说目录

# 目录
url = 'http://book.zongheng.com/showchapter/672340.html'  # 目录url
req = requests.get(url=url)
req.encoding = req.apparent_encoding
html = req.text
bshtml = BeautifulSoup(html, features="lxml")
bshtml = bshtml.find_all('ul', class_='chapter-list clearfix')  
a_bf = BeautifulSoup(str(bshtml), features="lxml")
a = a_bf.find_all('a')  # bshtml = bshtml.find_all('ul', class_='chapter-list clearfix')筛选后还没有得到我们想要的结果，具体为什么这么做，详细过程自己调试看看了解了。
# print(a)
for net in a[4:]:  # 前面是最新的章节，和后面的重复，故从4开始
    print(net.string, net.get('href'))

F12检查后发现章节名称和链接在<ul class="chapter-list clearfix"></ul>里：

a = a_bf.find_all('a')筛选目的：

最后效果图：

最终组合代码：

class downloader(object):
    def __init__(self):
        self.url = 'http://book.zongheng.com/showchapter/672340.html'
        self.names = []  # c存章节名
        self.urls = []  # 存章节链接
        self.nums = 0  # 章节数


    def get_download_url(self):
        req = requests.get(url=self.url)
        html = req.text
        bf = BeautifulSoup(html, features='lxml')
        div = bf.find_all('ul', class_='chapter-list clearfix')
        a_bf = BeautifulSoup(str(div), features="lxml")
        a = a_bf.find_all('a')
        self.nums = len(a[4:])
        for each in a[4:]:
            self.names.append(each.string)
            self.urls.append(each.get('href'))
        # print(self.urls)
        return self.urls, self.names


    def get_contents(self, url):
        # for url in self.urls:
        req = requests.get(url=url)
        html = req.text
        bf = BeautifulSoup(html, features="lxml")
        texts = bf.find_all('div', class_='content')
        texts = texts[0].text.replace('</p><p>', '\n\n')
        # print(self.names[i])
        # i += 1
        print(texts)
        return texts


if __name__ == '__main__':
    dl = downloader()
    dl.get_download_url()
    # dl.get_contents(dl.get_download_url())
    file_handle = open('剑来.txt', mode='w')
    for i in range(dl.nums):
        file_handle.write(dl.names[i])
        file_handle.write(dl.get_contents(dl.urls[i]))
        print("已完成第{}章小说爬取".format(i))
    file_handle.close()

效果图：