这里使用bs4的beautifulSoup,请求链接的使用使用try 以防止网络不稳定
#-*-coding:utf-8-*-
from urllib import request
from time import sleep
from bs4 import BeautifulSoup as btf
import os
header={
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0",
"Referer":"http://www.xxbiquge.com/3_3042"
}
def get_content(pagepath,filename): #通过url获取页面内容并写入到文件的函数
filename=filename+".txt" #组成文件名
url="http://www.xxbiquge.com/3_3042/"+pagepath #组成url
print("%s starts" %filename,url)
req=request.Request(url,data=None,headers=header)
fail=True
while(fail):
try:
res=request.urlopen(req).read()
res=res.decode('utf-8').replace('<br />','\n') #换行符转换
soup=btf(res,'lxml')
except Exception as e:
print(e)
sleep(1)
else:
fail=False
elements=soup.find_all('div',attrs={'id':'content'}) #匹配id='content'的内容
for i in elements:
with open(filename,'w',encoding='utf-8') as f:
f.write(i.text) #先把内容加码成utf8再写入文件
sleep(1)
if __name__=="__main__":
dirname = '民国投机者'
path = os.path.join(os.getcwd(), dirname) #本地路径拼接
if not os.path.exists(path): #文件夹是否存在,不存在就新建一个
os.mkdir(dirname)
os.chdir(path) #进入图片保存路径
url="http://www.xxbiquge.com/3_3042/" #入口页面
req=request.Request(url,data=None,headers=header)
fail=True
while(fail):
try:
ope=request.urlopen(req)
soup=btf(ope.read(),'lxml')
except Exception as e:
print(e)
sleep(1)
else:
fail=False
elements=soup.find_all("dd")
count=1
for i in elements:
pagepath=os.path.basename(i.a["href"])
get_content(pagepath,str(count)+'_'+i.a.text.split(' ')[1]) #url, filename
count+=1
效果如下