目的:爬取当当网中考研政治书籍中,每本书的书名、作者、商品网址、发布时间、出版社、好评率、评论数、现在价格、原先价格、折扣以及每本书简要的信息
结果呈现:存入txt中,屏幕以进度条下载的显示
注:马上考研了,考研党加油啊!!!考上研究生,你就发现研究生就是坑~~~来来来,我在坑里等你~~~
当当网搜索的网址:http://search.dangdang.com
#下面为本实例的爬虫代码,若有问题可以给我留言,或者有更好的解决方法也可以私信我~
一、不使用多线程
import requests from bs4 import BeautifulSoup import re import sys import time def get_page(url,params=None): headers={'user-agent':'Mozilla/5.0'} try: r=requests.get(url,headers=headers,params=params) r.raise_for_status() r.encoding=r.apparent_encoding return r.text except Exception as e: print(e) def get_pagenum(url):#得到用多少页 key='考研政治' params={ 'key':key, 'act':'input', } html=get_page(url,params) soup=BeautifulSoup(html,'html.parser') ul=soup.find('ul',{'name':{'Fy'}}) num_li=int(ul('li')[-3].text.strip()) return num_li def get_pageinfo(url):#得到每一页上多本书籍的信息 key = '考研政治' num=get_pagenum(url) print('爬取进度:') for i in range(num+1): jd=float((i+1)*100/num) #print('共{}页,当前爬取第{}页,进度:{}%'.format(num,i+1,jd)) sys.stdout.write(str(jd)+'%'+'→') sys.stdout.flush() params={ 'key':key, 'act':'input', 'page_index':str(i+1), } html=get_page(url,params) soup=BeautifulSoup(html,'html.parser') ul=soup.find('ul',{'class':{'bigimg'}}) for li in ul('li'): try: title=li('a')[0]['title'] #书名 #作者 p1=li.find('p',{'class':{'search_book_author'}}) if len(p1('span')[0]('a'))==1: author=p1('span')[0]('a')[0].text.strip() elif len(p1('span')[0]('a'))>1: author_list=[] for a in p1('span')[0]('a'): author_list.append(a.text.strip()) author='、'.join(author_list) else: author='无作者' publish_date=p1('span')[1].text.strip().split('/')[-1] #发布时间 publish_place=p1('span')[2].text.strip().split('/')[-1] #出版社 a1=li.find('a',{'name':{'itemlist-title'}}) book_url=a1['href']#商品网址 span1=li.find('span',{'class':{'search_star_black'}}) good_praise=span1('span')[0]['style'] good_praise=re.findall('.*?(\d+%);',good_praise)[0] ##好评率 li1=li.find('a',{'name':{'itemlist-review'}}).text.strip() comment=re.findall('(\d+)',li1)[0] now_price = li.find('span', {'class': {'search_now_price'}}).text.strip() # 现价 before_price=li.find('span',{'class':{'search_pre_price'}}).text.strip() #现价 discount=li.find('span',{'class':{'search_discount'}}).text.strip() discount=re.findall(r'\((.*)\)',discount)[0] #折扣 detail = li.find('p', {'class': {'detail'}}).text.strip() except: title='不详' author = '不详' publish_date= '不详' publish_place = '不详' book_url= '不详' good_praise= '不详' comment='不详' now_price = '不详' before_price = '不详' discount= '不详' detail = '不详' save_to_text(title,author,publish_date,publish_place,book_url,good_praise,comment,now_price,before_price,discount,detail) def save_to_text(title,author,publish_date,publish_place,book_url,good_praise,comment,now_price,before_price,discount,detail): with open('当当网.txt','a',encoding='utf-8')as f: f.write('书名:'+title+'\n') f.write('作者:'+author+'\n') f.write('商品网址:'+book_url+'\n') f.write('发布时间:'+publish_date+' '+'发布地址:'+publish_place+'\n') f.write('好评率:'+good_praise+' '+'评论数:'+comment+'\n') f.write('现在价格:'+now_price+' '+'原先价格:'+before_price+' '+'折扣:'+discount+'\n') f.write('详细信息:'+detail+'\n') f.write('❤❤❤❤❤❤❤❤❤❤❤❤') f.write('\n') f.close() if __name__ == '__main__': #key=input('请输入要搜索的书籍名称:{}').format().strip() 可以自己定义输入要搜索的书籍 start_url='http://search.dangdang.com' start_time=time.time() get_pageinfo(start_url) end_time=time.time() print('\n') print('花费时间:{}'.format(end_time-start_time))
(1)屏幕显示:
花费时间:699.6030657291412
(2)文本内容显示:
二、使用多线程
import requests from bs4 import BeautifulSoup import re import sys import time from multiprocessing import Pool def get_page(url,params=None): headers={'user-agent':'Mozilla/5.0'} try: r=requests.get(url,headers=headers,params=params) r.raise_for_status() r.encoding=r.apparent_encoding return r.text except Exception as e: print(e) def get_pagenum(url):#得到用多少页 key='考研政治' params={ 'key':key, 'act':'input', } html=get_page(url,params) soup=BeautifulSoup(html,'html.parser') ul=soup.find('ul',{'name':{'Fy'}}) num_li=int(ul('li')[-3].text.strip()) return num_li def get_pageinfo(i):#得到每一页上多本书籍的信息 url='http://search.dangdang.com' key = '考研政治' params={ 'key':key, 'act':'input', 'page_index':str(i+1), } html=get_page(url,params) soup=BeautifulSoup(html,'html.parser') ul=soup.find('ul',{'class':{'bigimg'}}) for li in ul('li'): try: title=li('a')[0]['title'] #书名 #作者 p1=li.find('p',{'class':{'search_book_author'}}) if len(p1('span')[0]('a'))==1: author=p1('span')[0]('a')[0].text.strip() elif len(p1('span')[0]('a'))>1: author_list=[] for a in p1('span')[0]('a'): author_list.append(a.text.strip()) author='、'.join(author_list) else: author='无作者' publish_date=p1('span')[1].text.strip().split('/')[-1] #发布时间 publish_place=p1('span')[2].text.strip().split('/')[-1] #出版社 a1=li.find('a',{'name':{'itemlist-title'}}) book_url=a1['href']#商品网址 span1=li.find('span',{'class':{'search_star_black'}}) good_praise=span1('span')[0]['style'] good_praise=re.findall('.*?(\d+%);',good_praise)[0] ##好评率 li1=li.find('a',{'name':{'itemlist-review'}}).text.strip() comment=re.findall('(\d+)',li1)[0] now_price = li.find('span', {'class': {'search_now_price'}}).text.strip() # 现价 before_price=li.find('span',{'class':{'search_pre_price'}}).text.strip() #现价 discount=li.find('span',{'class':{'search_discount'}}).text.strip() discount=re.findall(r'\((.*)\)',discount)[0] #折扣 detail = li.find('p', {'class': {'detail'}}).text.strip() except: title='不详' author = '不详' publish_date= '不详' publish_place = '不详' book_url= '不详' good_praise= '不详' comment='不详' now_price = '不详' before_price = '不详' discount= '不详' detail = '不详' save_to_text(title,author,publish_date,publish_place,book_url,good_praise,comment,now_price,before_price,discount,detail) def save_to_text(title,author,publish_date,publish_place,book_url,good_praise,comment,now_price,before_price,discount,detail): with open('当当网_多线程.txt','a',encoding='utf-8')as f: f.write('书名:'+title+'\n') f.write('作者:'+author+'\n') f.write('商品网址:'+book_url+'\n') f.write('发布时间:'+publish_date+' '+'发布地址:'+publish_place+'\n') f.write('好评率:'+good_praise+' '+'评论数:'+comment+'\n') f.write('现在价格:'+now_price+' '+'原先价格:'+before_price+' '+'折扣:'+discount+'\n') f.write('详细信息:'+detail+'\n') f.write('❤❤❤❤❤❤❤❤❤❤❤❤') f.write('\n') f.close() if __name__ == '__main__': start_url='http://search.dangdang.com' start_time = time.time() num=get_pagenum(start_url) pool=Pool() pool.map(get_pageinfo, [i for i in range(num+1)]) end_time=time.time() print('\n') print('花费时间:{}'.format(end_time-start_time))
(1)屏幕显示
花费时间:267.2782230377197
(2)文本内容显示
对比:
不使用多线程花费时间:699.60秒
使用多线程发费时间:267.28秒
在本例子中,使用多线程,提高了2.6倍的时间!!!!!
所以...............多线程好啊!!!
感悟:学习也是这样,只有找对了方法,我们的效率得到提高,事半功倍~~~
今日爬虫完成!
今日鸡汤:所谓的勇气,是清楚地知道一切好运都不会从天而降,只有不断努力争取!
加油ヾ(◍°∇°◍)ノ゙