爬取静态网页之爬虫小说
复现爬虫小说
这里不做理论介绍,可以移步这里爬虫理论学习或者自己搜。
目标URL:笔趣阁
引入库:
import sys
import requests
from bs4 import BeautifulSoup
看看小说的正文
url = 'https://www.bqkan8.com/1_1094/5403177.html' # 目标url
response = requests.get(url=url) # 获取网页HTML
html = response.text # .text 返回的是Beautifulsoup根据猜测的编码方式将content内容编码成字符串。
bf = BeautifulSoup(html, features="lxml") # 不写features="lxml",会warning:BeautifulSoup库未写明解析器警告
texts = bf.find_all('div', class_='showtxt') # 在bf中寻找标签为div,class为showtxt的语句
texts = texts[0].text.replace('\xa0'*8, '\n\n') # 替换
print(texts)
代码结果:
小说所有目录
# 目录
server = 'https://www.bqkan8.com/'
url = 'https://www.bqkan8.com/1_1094/'
response = requests.get(url)
response.encoding = response.apparent_encoding # 解决返回字符串乱码问题
html = response.text
bshtml = BeautifulSoup(html, features="lxml")
bshtml = bshtml.find_all('div', class_='listmain')
a_bf = BeautifulSoup(str(bshtml), features="lxml")
a = a_bf.find_all('a')
for each in a[13:]:
print(each.string, server + each.get('href'))
代码结果:
最终组合代码:
class downloader(object):
def __init__(self):
self.server = 'https://www.bqkan8.com/'
self.url = 'https://www.bqkan8.com/1_1094/'
self.names = [] # 存放章节名
self.urls = [] # 存放章节链接
self.nums = 0 # 章节数
def get_download_url(self):
req = requests.get(url=self.url)
html = req.text
bf = BeautifulSoup(html, features="lxml")
div = bf.find_all('div', class_='listmain')
a_bf = BeautifulSoup(str(div), features="lxml")
a = a_bf.find_all('a')
self.nums = len(a[13:])
for each in a[13:]:
self.names.append(each.string)
self.urls.append(self.server+each.get('href'))
print(self.urls)
return self.urls
def get_contents(self, url):
req = requests.get(url=url)
html = req.text
bf = BeautifulSoup(html, features="lxml")
texts = bf.find_all('div', class_='showtxt')
texts = texts[0].text.replace('\xa0'*8, '\n\n')
# print(texts)
return texts
def wrist(self, name, path, text):
with open(path, 'a', encoding='utf-8') as f:
f.write(name+'\n')
f.writelines(text)
f.write('\n\n')
if __name__ == "__main__":
dl = downloader()
dl.get_download_url()
# dl.get_contents(dl.get_download_url())
print('《一年永恒》开始下载:')
for i in range(dl.nums):
dl.wrist(dl.names[i], '一念永恒.txt', dl.get_contents(dl.urls[i]))
print("已完成第{}章小说爬取".format(i))
print('《一年永恒》下载完成')
代码结果:
参考博客:复现的文章出处,非常值得一看
自己的爬虫小说
思路:
- 拿到所有章节名和章节链接
- 根据章节链接找到每章小说正文
- 根据章节名分类保存小说
# 引入库
import requests
from bs4 import BeautifulSoup
看看小说正文
# 每章小说正文,以第一章为例:
url = 'http://book.zongheng.com/chapter/672340/36898237.html' # 第一章小说正文的url
req = requests.get(url=url)
html = req.text
bf = BeautifulSoup(html, features="lxml")
texts = bf.find_all('div', class_='content')
texts = texts[0].text.replace('</p><p>', '\n\n') # 把</p><p>替换成换行
print(texts)
F12检查后发现,正文在
里:![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/56ad7396ccb2ae00706fa677bf3da474.png)
效果图:
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/6ab79009254bc2f6c795213c48ecc7da.png)
小说目录
# 目录
url = 'http://book.zongheng.com/showchapter/672340.html' # 目录url
req = requests.get(url=url)
req.encoding = req.apparent_encoding
html = req.text
bshtml = BeautifulSoup(html, features="lxml")
bshtml = bshtml.find_all('ul', class_='chapter-list clearfix')
a_bf = BeautifulSoup(str(bshtml), features="lxml")
a = a_bf.find_all('a') # bshtml = bshtml.find_all('ul', class_='chapter-list clearfix')筛选后还没有得到我们想要的结果,具体为什么这么做,详细过程自己调试看看了解了。
# print(a)
for net in a[4:]: # 前面是最新的章节,和后面的重复,故从4开始
print(net.string, net.get('href'))
F12检查后发现章节名称和链接在<ul class="chapter-list clearfix"></ul>
里:
a = a_bf.find_all('a')
筛选目的:
最后效果图:
最终组合代码:
class downloader(object):
def __init__(self):
self.url = 'http://book.zongheng.com/showchapter/672340.html'
self.names = [] # c存章节名
self.urls = [] # 存章节链接
self.nums = 0 # 章节数
def get_download_url(self):
req = requests.get(url=self.url)
html = req.text
bf = BeautifulSoup(html, features='lxml')
div = bf.find_all('ul', class_='chapter-list clearfix')
a_bf = BeautifulSoup(str(div), features="lxml")
a = a_bf.find_all('a')
self.nums = len(a[4:])
for each in a[4:]:
self.names.append(each.string)
self.urls.append(each.get('href'))
# print(self.urls)
return self.urls, self.names
def get_contents(self, url):
# for url in self.urls:
req = requests.get(url=url)
html = req.text
bf = BeautifulSoup(html, features="lxml")
texts = bf.find_all('div', class_='content')
texts = texts[0].text.replace('</p><p>', '\n\n')
# print(self.names[i])
# i += 1
print(texts)
return texts
if __name__ == '__main__':
dl = downloader()
dl.get_download_url()
# dl.get_contents(dl.get_download_url())
file_handle = open('剑来.txt', mode='w')
for i in range(dl.nums):
file_handle.write(dl.names[i])
file_handle.write(dl.get_contents(dl.urls[i]))
print("已完成第{}章小说爬取".format(i))
file_handle.close()
效果图: