参考其他代码,用python3写的一个简单demo,结构比较清晰,适合入门。
# 引入库
import requests
import re #正则表达式
# 写网站站点
url = "http://www.jingcaiyuedu.com/novel/GLSmM4.html"
# 写入headers模拟浏览器上网,避免出现个别网站拒绝访问的情况
headers = {"user-agent": "Mozilla/5.0"}
# get发送请求
response = requests.get(url,headers=headers)
# 将网页编码方式转换为utf-8
response.encoding = 'utf-8'
# 网站源码
html = response.text
# re.findall获取小说的名字
title = re.findall(r'<meta property="og:title" content="(.*?)"/>',html)[0]
# 获取每一章的信息(章节的url)
dl = re.findall(r'<dl class="panel-body panel-chapterlist">.*?</dl>',html,re.S)[1]
aill = re.findall(r'href="(.*?)">(.*?)<',dl)
# 新建文件保存小说内容
f = open("%s.txt" % title, 'w', encoding="utf-8")
# 循环每一个章节,分别去下载
for i in aill:
# 章节地址和名
book_url,book_name = i
print(book_url)
# 正确章节地址http://www.jingcaiyuedu.com/novel/GLSmM4/1.html
# 拼接正确章节地址
book_url = "http://www.jingcaiyuedu.com%s" % book_url
# 获取章节
book_response = requests.get(book_url, headers=headers)
book_response.encoding = 'utf-8'
book_html = book_response.text
# 提取章节内容
book_content = re.findall(r'<div class="panel-body" id="htmlContent">(.*?)</div>', book_html, re.S)[0]
# 清洗提取的数据
book_content = book_content.replace(' ', '')
# 将其中内容的空格部分替换成空
book_content = book_content.replace(' ', '')
# 将其中内容的 部分替换成空
book_content = book_content.replace('<br />', '')
# 将其中内容的<br />部分替换成空
book_content = book_content.replace('<br/>', '')
# 将其中内容的<br/>部分替换成空
# 写入
f.write(book_name)
f.write(book_content)
f.write("\n")
print(book_url)