import requests
import re
import os
url = 'url'
html = requests.get(url).text
def get_cahpter_url(html):
"""
获取每一章的链接,储存到一个列表里并返回
:param html: 目录页源代码
:return: 每章链接
"""
chapter_url_list = []
chapter_block_big = re.findall('<div id="play_0">(.*?)</ul>', html, re.S)[0]
chapter_block_little = re.findall('href="(.*?)"', chapter_block_big, re.S)
for i in chapter_block_little:
chapter_url_list.append('url'+i)
return chapter_url_list
def get_content(html):
"""
获取每一章的正文并返回章节名和正文
:param html: 正文源代码
:return: 章节名,正文
"""
content_list_big = re.findall('>www</span>(.*?)<div class=', html, re.S)[0]
content_list_little = re.findall('<p>(.*?)</p>', content_list_big, re.S)
content_topic = re.search('<h1>(.*?)</h1>', html, re.S).group(1)
content = ""
for i in content_list_little:
content = content + i + "\n"
return content_topic,content
def save(chapter, article):
"""
将每一章保存到本地
:param chapter:章节名
:param content: 第x章
:return: None
"""
os.makedirs('文件名', exist_ok=True)
#如果没有“诡文件名”文件夹,就创建一个,如果有,就什么都不做
with open(os.path.join('文件名',chapter+'.txt'), 'w', encoding='utf-8') as f:
f.write(article)
return None
chapter_url_list = get_cahpter_url(html)
for i in chapter_url_list:
requests.get(i).encoding='UTF-8'
chapter_url=requests.get(i).content.decode('utf-8')
a=get_content(chapter_url)
save(a[0], a[1])
print('下载完成')
学爬虫的代码,搞了将近两周,终于爬出来了,真不容易啊......