#爬取单章节的文字内容
from urllib import request
from bs4 import BeautifulSoup
from urllib.error import HTTPError
#第6章的网址
def book(i):
url='https://dldl1.nsbuket.cc/xiaoshuo/douluodalu/{}.html'.format(i)
head={}
head['User-Agent']='https://dldl1.nsbuket.cc/xiaoshuo/douluodalu/{}.html'.format(i)
try:
reg =request.Request(url,headers=head)
response = request.urlopen(reg)
html = response.read()
# 创建 request 对象
soup = BeautifulSoup(html, 'lxml')
title = soup.find(name='h1')
soup_text = soup.find(name='div', attrs={"class": "m-post"})
return title,soup_text
except HTTPError as e:
if e.code == 404:
return None
file = open(r'斗罗大陆.txt', 'w',encoding='utf-8')
a=0
for i in range(1,218020):
bo=book(i)
if bo is not None: # 检查book_text是否不是None
ti = book(i)[0].text
te=book(i)[1].text
a=a+1
#b='第{}章'.format(a)
#file.write(b)
#for bo in bo:
print(ti,te)
file.write('\n'+ti + '\n'+ te + '\n')
# 输出文本内容
else:
continue
#if reg.status_code == 404:
#continue
#并输出其中的文本
file.close()
可以爬标题,文本内容,用了一个报错防止网页404,存到记事本里