该文章是自己在初学爬虫时,选择了 看书海 这个网站进行爬虫,慢慢地学习新的内容(多线程,代理).
整体思路为四步走
一、页面爬虫
连载书库 这个链接是文章列表,该站所有的小说都在里面. 共有2000多页. 目的就是获取到2000多页的链接.
二、书本链接爬虫
根据第一步获得的2000多个链接,爬取其书本的URL
三、单个书本所有章节爬虫
根据第二步爬取的书本URL,爬取每一本书的每一个单章的链接
四、单章爬虫(代理+多线程)
利用上面获得的章节链接,1个1个进行爬虫.
一、爬取书库所有页面,并储存到数据库
创建一个数据库表,结构如图.
import requests
from bs4 import BeautifulSoup
from lxml import etree #用来计算sel和result. 及解析html中的网址路径
import time
import datetime
import numpy
import mysql.connector
database = mysql.connector.connect(user='XXXXX', password='XXXXX', database='看书海')
cursor = database.cursor()
headers = {'content-type': 'application/json',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'}
def get_pageurl():
global url
while True:
r = requests.get(url,headers=headers)
html = r.text.encode(r.encoding).decode('GB18030')
soup= BeautifulSoup(html,'lxml')
soup=soup.find("div", {"class":"pagelink","id": "pagelink"})
sel = etree.HTML(str(soup))
result = set(sel.xpath('//a/@href'))
for row in result:
try:
cursor.execute("INSERT INTO PageList(pageid,pageurl) value(%s,%s)",(row[row.find('_')+1:row.find('.html')],str(row)))
database.commit()
print(row[row.find('_')+1:row.find('.html')],'done')
except :
print(row[row.find('_')+1:row.find('.html')],'pass')
continue
url = sel.xpath('//a[@class="ngroup"]/@href')[0] #url为 > 箭头所指的链接.这样子可以少爬两次.
time.sleep(numpy.random.randint(1,5))
print('done')
falltimes = 0
def start():
global falltimes
try:
get_pageurl()
except:
falltimes +=1
print('fall',falltimes,'次')
time.sleep(10)
start()
结果如图..请不要吐槽其中的规律...
二、查询书本链接
import requests
from bs4 import BeautifulSoup
from lxml import etree #用来计算sel和result. 及解析html中的网址路径
import time
import datetime
import numpy
import mysql.connector
database = mysql.connector.connect(user='root', password='啊啊啊啊', database='看书海')
cursor = database.cursor()
headers = {'content-type': 'application/json',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'}
def get_bookurl():
cursor.execute("SELECT * FROM pagelist WHERE ISNULL(finished)")
pagelist = cursor.fetchall()
print('数据库读取完成!')
print('从page',pagelist[0][0],'开始')
for row in pagelist:
time_1 = datetime.datetime.now()
url = row[1]
pageid=row[0]
r = requests.get(url,headers=headers)
html = r.text.encode(r.encoding).decode('GB18030')
soup= BeautifulSoup(html,'lxml')
soup=soup.find("div", {"id":"bookinfo"})
sel = etree.HTML(str(soup))
result = (sel.xpath('//dt//a[@target="_blank"]/@href'))
for row in result:
try:
bookid= row[row.find('/',row.find('/',int(row.find('/',10)+1)+1)+1)+1:
row.find('/',row.find('/',row.find('/',int(row.find('/',10)+1)+1)+1)+1)]
cursor.execute("INSERT INTO booklist(pageid,bookid,bookurl) value (%s,%s,%s)",(pageid,bookid,str(row)))
#将bookid和bookurl储存到数据库. bookid是唯一的.
database.commit()
except:
print(bookid,'pass')
continue
time.sleep(numpy.random.randint(1,3))
print('完成了page',pageid,'耗时',(datetime.datetime.now()-time_1).seconds,'s')
cursor.execute("UPDATE pagelist SET finished = %s where pageid = %s ",(1,pageid))
import traceback #用作报错追踪
def start():
falltimes=0
try:
get_bookurl()
except:
traceback.print_exc()
falltimes += 1
print('fall',falltimes,'times')
time.sleep(100)
start()
time_start = datetime.datetime.now()
start()
print('总耗时',datetime.datetime.now()-time_start)
如上图. 不过这里的简介/作者,是在第三部添加上去的.
三、查询章节链接
这是最后一个表. chapterlist.
import requests
from bs4 import BeautifulSoup
from lxml import etree #用来计算sel和result. 及解析html中的网址路径
import time
import datetime
import numpy
import mysql.connector
headers = {'content-type': 'application/json',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'}
#邮箱提醒
from email import encoders
from email.header import Header
from email.mime.text import MIMEText
from email.utils import parseaddr, formataddr
import smtplib
def _format_addr(s):
name, addr = parseaddr(s)
return formataddr((Header(name, 'utf-8').encode(), addr))
def send_mail(sendMessage):
from_addr = '啊啊啊@163.com'
password = '啊啊啊啊'
to_addr = '啊啊啊啊@foxmail.com'
smtp_server = 'smtp.163.com'
sendTitle = 'GET CHAPTER 爬取失败!'
sendText = sendMessage
msg = MIMEText(sendText, 'plain', 'utf-8')
msg['From'] = _format_addr('Jack<%s>' % from_addr)
msg['To'] = _format_addr(' <%s>' % to_addr)
msg['Subject'] = Header(sendTitle, 'utf-8').encode()
server = smtplib.SMTP(smtp_server, 25)
server.login(from_addr, password)
server.sendmail(from_addr, [to_addr], msg.as_string())
server.quit()
#通过链接booklist,获取单本书的链接.并获得author/bookname/introduction.
#finished=1代表完成. finished=2,代表getUrl未知错误. finished=3,代表UnicodeDecodeError
def get_book():
database = mysql.connector.connect(user='root', password='yituba', database='看书海')
cursor = database.cursor()
global n
cursor.execute("SELECT * FROM booklist where isnull(finished) ORDER BY bookid ;")
booklist = cursor.fetchall()
for book in booklist:
time_1 = datetime.datetime.now()
url=book[2]
bookid=book[1]
print(url,end=' ')
try:
r = requests.get(url,headers=headers,timeout=20)
html = r.text.encode(r.encoding).decode('GB18030')
except UnicodeDecodeError:
print('UnicodeDecodeError !! continue')
cursor.execute("UPDATE booklist SET finished = 3 WHERE bookid=%s"%str(bookid))
database.commit()
send_mail('UnicodeDecodeError')
continue
except requests.exceptions.ConnectTimeout :
print('Time out')
send_mail('ConnectTimeout. Sleep 500 seconds')
time.sleep(500)
except :
print('request url falls. sleep 300 seconds')
cursor.execute("UPDATE booklist SET finished = 2 WHERE bookid=%s"%str(bookid))
database.commit()
text2 = traceback.format_exc()
send_mail(text2)
time.sleep(300)
continue
soup= BeautifulSoup(html,'lxml')
#书名
bookname = soup.find("div", {"id":"title"}).string[0:soup.find("div", {"id":"title"}).string.find('全文阅读')]
#书籍简介
introction_list =[]
for row in soup.find("div", {"class":"intro"}).stripped_strings:
introction_list.append(row)
introction=''.join(introction_list[1:]) #将list转化为字符串. 同时忽略第一个"内容简介:"
#作者
author_list=[]
for row in soup.find("div", {"class":"rep"}).strings:
author_list.append(row)
author = author_list[1]
#将书名/简介/作者传入数据库
cursor.execute("UPDATE booklist SET bookname=%s,introduction=%s,bookauthor=%s WHERE bookid=%s",
(str(bookname),str(introction),str(author),bookid))
database.commit()
#chapterlink
sel = etree.HTML(str(soup.find("div",{"id":"book"})))
result = sel.xpath('//@href')
chapter=[]
n=1
for row in result:
chapterurl = url+row
bookid_str=str(bookid).zfill(5)
n_str=str(n).zfill(5)
chapter_id = '%s%s'%(bookid_str,n_str)
try:
cursor.execute("INSERT chapterlist(bookid,chapterid,chapterurl) value (%s,%s,%s)",(bookid,chapter_id,chapterurl))
except mysql.connector.errors.IntegrityError :
print(chapter_id,'已存在')
continue
n=n+1
cursor.execute("UPDATE booklist SET finished = 1 WHERE bookid=%s"%str(bookid))
database.commit()
print(bookid,bookname,'完成! 耗时',(datetime.datetime.now()-time_1).seconds,'seconds',' 共有',n,'章')
time.sleep(numpy.random.randint(0,5))
database.close()
import traceback
def start():
try:
get_book()
except:
sendMessage = '错误代码如下:\n'+traceback.format_exc()
print('出现问题.等待100s后重新开始')
send_mail(sendMessage)
time.sleep(100)
start()
start()
这里比起之前主要是多了一个报错信息邮箱提醒. 这样方便让自己还知道爬虫是不是还在跑..
这个大概一两天就能跑完,一共是2000W行数据.大概就是2000万个章节.
(接下来需要爬虫..爬完这2000万个章节!!)
四、获取正文(代理+多线程)
获取正文这里就是大头了. 因为上面提到有2000万个url需要爬... 如果还是想上面那样,用sleep大法来的话,要爬好几年...
所以用代理,防止被封ip. 同时用多线程,加快访问速度.(反正都是在等I/O,不如大家一起跑)
import mysql.connector
database = mysql.connector.connect(user='root', password='啊啊啊', database='看书海')
cursor = database.cursor()
import datetime
import time
import threading
import requests
from bs4 import BeautifulSoup
from lxml import etree #用来计算sel和result. 及解析html中的网址路径
import traceback
headers = {'content-type': 'application/json',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'}
#邮箱提醒
from email import encoders
from email.header import Header
from email.mime.text import MIMEText
from email.utils import parseaddr, formataddr
import smtplib
def _format_addr(s):
name, addr = parseaddr(s)
return formataddr((Header(name, 'utf-8').encode(), addr))
def send_mail(sendTitle,sendMessage):
try:
from_addr = '啊啊啊@163.com'
password = '啊啊啊啊'
to_addr = '啊啊啊@foxmail.com'
smtp_server = 'smtp.163.com'
sendText = sendMessage
msg = MIMEText(sendText, 'plain', 'utf-8')
msg['From'] = _format_addr('Jack<%s>' % from_addr)
msg['To'] = _format_addr(' <%s>' % to_addr)
msg['Subject'] = Header(sendTitle, 'utf-8').encode()
server = smtplib.SMTP(smtp_server, 25)
server.login(from_addr, password)
server.sendmail(from_addr, [to_addr], msg.as_string())
server.quit()
except:
print('发送邮件失败')
#这个函数主要是通过公式来算出proxies.方便下面利用requests进行使用.所以也要加上global
def get_proxies():
global proxies
# 要访问的目标页面
targetUrl = "http://test.abuyun.com"
# 代理服务器
proxyHost = "http-pro.abuyun.com"
proxyPort = "9010"
# 代理隧道验证信息
proxyUser = "啊啊啊啊" #来自于阿里布. 这是供应商提供的.
proxyPass = "啊啊啊啊"
proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {
"host" : proxyHost,
"port" : proxyPort,
"user" : proxyUser,
"pass" : proxyPass,
}
proxies = {
"http" : proxyMeta,
"https" : proxyMeta,
}
以上是一些比较基本的公式引用.
def get_list():
global value_database
cursor.execute("SELECT c.bookid,c.chapterid,c.chapterurl,b.bookname FROM chapterlist as c \
left join booklist as b on b.bookid = c.bookid where isnull(c.chaptername) and isnull(c.iserror) limit 4")
value_database= cursor.fetchall()
print('读取数据库完成')
#litmit 4 是为了限制多线程的数量... 之所以是4,不是因为四核,而是因为对方代理只允许1秒内访问4次..
error_number=0
def get_content(chapterid,url):
global allcontent_list,error_number
try: #利用try..except 来规避一些异常数据
r = requests.get(url,headers=headers,timeout=10,proxies=proxies)
html = r.text.encode(r.encoding).decode('gb18030')
except requests.exceptions.ReadTimeout:
print(chapterid,'ReadTimeout',end=" ")
error_number+=1
except requests.exceptions.ConnectionError:
print(chapterid,'ConnectionError',end=" ")
error_number+=1
except UnicodeDecodeError: #有些页面就是解码有问题..就是对方网页问题.这种直接添加一个iserror的异常,保证接下来不会被重复引用链接.
cursor.execute("UPDATE chapterlist SET iserror = 1 where chapterid = %s"%(chapterid))
print('解码错误')
error_number+=1
except TypeError:
print(chapterid,'TypeError')
error_number+=1
except:
sendMessage = str(chapterid)+'错误代码如下:\n'+traceback.format_exc()
send_mail("看书海多线程失败",sendMessage)
print(chapterid,'出错',end=" ")
else:
soup= BeautifulSoup(html,'lxml')
chapter_content_html = soup.find("div",{"id": "con_L"})
content=' '
for string in chapter_content_html.stripped_strings:
content = content + string +'\n' + ' '
charpter_name=soup.h1.string
content_list=[str(charpter_name),str(content),int(chapterid)]
allcontent_list.append(content_list)
get_list(): 数据库中获得需要爬取的URL. 同时也是为了保证不会重复
get_content(chapterid,url): 通过chapterid和url直接获得content并存到数据库中.
设置多线程
def multi():
global allcontent_list
allcontent_list=[]
threads = []
for row in value_database:
chapterid = row[1]
url=row[2]
t=threading.Thread(target=get_content,args=(chapterid,url))
threads.append(t)
if __name__ == '__main__':
for t in threads:
t.setDaemon(True)
t.start()
for t in threads:
t.join()
#以下的考虑主要是多线程数据库读写会导致其报错. 因此将多线程的结果存在list中,等到多线程结束以后,再进行数据库读写.这样就不会错啦.
for row in allcontent_list:
cursor.execute("UPDATE chapterlist SET chaptername = %s ,content = %s where chapterid = %s",
(row[0],row[1],row[2]))
database.commit()
print('传入数据库完成')
n=0 #报错次数
get_proxies() #获取代理的链接
while True:
t1=datetime.datetime.now()
get_list()
multi()
n+=1
print('已完成',n,'All done.耗时',(datetime.datetime.now()-t1).seconds,'秒',"失败了"+str(error_number)+"次")
if n%200==0: #每200次进行一次报错.提醒自己进度如何了
sendMessage='已完成'+str(n)+'次'+'失败了'+str(error_number)+'次'
send_mail("看书海多线程进度汇报",sendMessage)
以上爬虫写得非常粗糙,但还是能基本上把看书海这个网站的小说爬下来. (虽然我还没成功,太慢了...)
代理是付费的,1块钱1个小时,我爬了一个小时就不爬了,免得对服务器造成冲击.