Python爬取小说
这里主要爬取笔趣阁的链接
因为笔趣阁对段时间的爬取次数做了限制,所以每次我们只能爬取十章
# coding:utf-8
import re
import soup as soup
from bs4 import BeautifulSoup
import requests
import chardet
i=0
# #限定每行字数
def cut_text(text,lenth):
textArr = re.findall('.{'+str(lenth)+'}', text)
textArr.append(text[(len(textArr)*lenth):])
return textArr
def read(url, i):
i+=1
if i>10:
return
r = requests.get(url)
encode=chardet.detect(r.content)
r.encoding=encode['encoding']
doc = r.text # 服务器返回响应
soup = BeautifulSoup(doc, "html.parser")
for title in soup.find_all('h1'):
print(" ")
print(title.string) # 输出响应的html对象
for txt in soup.find_all(id='content'):
# #每行字数限定
# #笔趣阁1
# for line in txt.text.split(' '):
# if len(line)>40:
# for cutline in cut_text(line,40):
# print(cutline)
# else:print(line)
#笔趣阁2
for line in txt.text.split():
if len(line)>40:
for cutline in cut_text(line,40):
print(' '+cutline)
else:print(' '+line)
# #笔趣阁1
# for c in soup.find_all(class_='page_chapter'):
# d= c.find_all('a')
# href=d[2]['href']
#笔趣阁2-1
for c in soup.find_all(id='pager_next'):
href=c['href']
# #笔趣阁2-2
# for c in soup.find_all(class_='bottem2'):
# d= c.find_all('a')
# href=d[3]['href']
if len(href)>2:
# #笔趣阁1
# href='https://www.biqukan.com'+href
# 笔趣阁2-1
href='https://www.xinxs.la/489_489289/'+href
# # 笔趣阁2-2
print(href)
# href='https://www.guibuyu.org/'+href
read(href,i)
# #这里需要持续更新链接,
# #笔趣阁1 最初进化
# url='https://www.biqukan.com/75_75537/521520773.html'
# 笔趣阁2-1
url = 'https://www.xinxs.la/489_489289/2647622.html'
# # 笔趣阁2-2
# url = 'https://www.guibuyu.org/96_96741/33481914.html'
read(url,i)