网络爬虫二

以抓取豆瓣电影前250为例，如何从下载的网页提取数据

提取数据主要有三种方法 ，正则表达式，beautiful soup 和lxml

首先查看网页代码构造

利用开发人员工具

 

# coding:utf-8

import requests
from bs4 import BeautifulSoup
import re
import time
import sys
import io
import bs4
import lxml.html
from pymongo import MongoClient

def req(url, param):
	resp = requests.get(url, params=param).text
	return resp
	

def getHTMLText(url,k):
	#https://movie.douban.com/top250?start=25&filter=
	try:
		if(k==0):
			kw={}
		else:
			kw={'start':k,'filter':''}
		#获取网页
		#params 以字符串字典的形式传递参数
		r = requests.get(url,params=kw,headers={'User-Agent': 'Mozilla/4.0'})
		r.raise_for_status()#检测响应状态码
		r.encoding = r.apparent_encoding
		return r.text
	except:
		print("Failed!")

def getData(html):
	soup = BeautifulSoup(html, "html.parser")
	#data_ol = soup.ol
	#films = []
	#for tag_li in data_ol:
	#	films = []
	#	if isinstance(tag_li,bs4.element.Tag): #判断一个对象是否是一个已知类型
	#		datas = lxml.html.fromstring(str(tag_li.contents))
	#		names=[]
	#		name1 = datas.xpath('//span[@class="title"]/text()')
	#		name2 = datas.xpath('//span[@class="other"]/text()')
	#		names.append(name1)
	#		names.append(name2)
	#		star = datas.xpath('//span[@class="rating_num"]/text()')	
	#		num = re.search('<span>(.*)</span>',str(data_ol.contents)).group(1)		
	#		quote = datas.xpath('//span[@class="inq"]/text()')
	#		film_info ={
	#		    'name': names,		    
	#		    'star': star,
	#		    'num':num,
	#		    'quote':quote
	#		}
	#		films.append(film_info)		
	#return films		
	#找到第一个class属性值为grid_view的ol标签
	movieList=soup.find('ol',attrs={'class':'grid_view'})
	
	moveInfo=[]
	for movieLi in movieList.find_all('li'):#找到li标签
		data = []
	#得到电影名称	
		#找到第一个class属性为hd的div标签
		movieHd = movieLi.find('div',attrs={'class':'hd'})
		#找到第一个class属性值为title的span标签
		movieName = movieHd.find('span',attrs={'class':'title'}).getText()	
		data.append(movieName)		
	#得到电影的评分
		movieScore = movieLi.find('span',attrs={'class':'rating_num'}).getText()
		data.append(movieScore)	
	#得到电影的评价人数
		movieEval=movieLi.find('div',attrs={'class':'star'})
		movieEvalNum=re.findall(r'\d+',str(movieEval))[-1]
		data.append(movieEvalNum)		
	#得到电影短评
		movieQuote = movieLi.find('span', attrs={'class':'inq'})
		if(movieQuote):
			data.append(movieQuote.getText())
		else:
			data.append("无")
		print(outputMode.format(data[0],data[1],data[2],data[3],chr(122)))

#将输出重定向到txt文件
output=sys.stdout
outputfile=io.open('moviedata.txt','w',encoding='utf-8')
sys.stdout=outputfile
reload(sys)
sys.setdefaultencoding('utf8')
outputMode = u"{0:{4}^20}\t{1:^10}\t{2:^10}\t{3:{4}<10}"
print(outputMode.format('电影名字','评分','评论人数','短评',chr(122)))
		
basicUrl='https://movie.douban.com/top250'

k=0
while k<=250:
	html=getHTMLText(basicUrl,k)
	time.sleep(2)
	k+=25
	getData(html)
	
outputfile.close()
sys.stdout=output	

注释掉的部分是采用lxml 的xpath选择器方法，当然也可以采用css选择器的方法

 

例如：