import requests
import re
import time
import os
#使用正则表达式的方法,少了解析数据这一步。通过requests库返回请求的HTML文件就是字符串的类型,可以直接提取数据
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'}
##获取信息
def get_info(url):
r = requests.get(url,headers=headers)
infos = re.findall("<p>(.*?)</p>",r.content.decode("utf-8"),re.S) #re.S 匹配包括换行在内的所有字符
return infos
##建立目录
def mkdir(path):
path = path.strip()
isExists = os.path.exists(path)
if not isExists:
os.mkdir(path)
else:
pass
if __name__ == "__main__":
r = requests.get("http://www.quanshuwu.com/book/2039.aspx",headers=headers)
base_url = "http://www.quanshuwu.com"
contents = re.findall('<li><a href="(.*?)">(.*?)</a></li>',r.content.decode("utf-8"))
for content in contents:
target_url = content[0]
name = content[1]
url = base_url + target_url
mkdir("悍妃")
with open("E:\python爬虫\悍妃天下\悍妃/" + name + ".txt","w") as f:
f.writelines(get_info(url))
time.sleep(1)
小说悍妃天下爬取 正则表达式
最新推荐文章于 2019-07-15 18:20:26 发布