爬取js渲染过的页面（爬取一个婚庆网站为例）

  
  

   
   这个网站是js渲染过的，所以我们可以使用PhantomJS浏览器或者在network中找出需要post的qurrystring中的参数，发请求就可以了，得到的是json
  
  
  
  

   
   # !/usr/bin/python
# -*- encoding: UTF-8 -*-
from lxml import etree
import urllib
import urllib2
import jsonpath
import json
from lxml import etree
class we():
	def __init__(self):
		self.page=3
		self.headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36",}
	def meiyiye(self):
		item=[]
		headers=self.headers
		url="http://search.jiayuan.com/v2/search_v2.php"
		fromdata={
		"sex":"f",
		"key":"",
		"stc":"1%3A41%2C2%3A19.27%2C3%3A155.170%2C23%3A1",
		"sn":"default",
		"sv":"1",
		"p":self.page,#表示爬取的第几页
		"f":"",
		"listStyle":"bigPhoto",
		"pri_uid":"170703614",
		"jsversion":"v5"}
		data = urllib.urlencode(fromdata)
		request=urllib2.Request(url,data=data,headers=headers)#post请求的话需要data值，而get请求不需要data有值
		response = urllib2.urlopen(request)
		#得到的是json格式的字符串（字典行的）
		html = response.read()
		html1=html.replace("##jiayser##","").replace("//","")
		#把json转换成python格式的unicode字符串（列表形式的）
		content = json.loads(html1)
		id_list=jsonpath.jsonpath(content,"$..uid")#在content中匹配出需要的个人id，然后通过这个id拼接出个人主页的链接
		for i in id_list:
			item.append(i)
		self.page+=1
		self.meiyigeren(item)

#处理这一页每一个人的页面信息
	def meiyigeren(self,item):
		for id in item:
			print "*******************************************"
			print u"用户id:"+str(id)
			url="http://www.jiayuan.com/"+str(id)+"?fxly=search_v2_index"#拼接连接，然后发送请求，找到个人主页中需要的有用的内容
			print u"主页链接:"+url
			headers=self.headers
			request=urllib2.Request(url,headers=headers)
			response = urllib2.urlopen(request)
			html = response.read()
			content=etree.HTML(html)#解析HTML文档为HTML DOM模型，然后下面就可以使用xpath匹配出想要的内容
			username=content.xpath('//div[@class="main_1000 bg_white mt15"]//h4/text()')
			if len(username)==1:
				print username[0]
			else:
				print u"没有名字"
			a=content.xpath('//div[@class="main_1000 bg_white mt15"]//h6[@class="member_name"]/text()')
			we=" ".join(a)
			ni=we.replace("，"," ").replace(',',' ')
			ha=ni.split(" ")
			print u"年龄:"+ha[0]
			header_url=content.xpath('//div[@class="big_pic fn-clear"]//li[2]//tr//img[@class="img_absolute"]//@_src')
			if len(header_url)==1:
				header_urll=header_url[0]
			else:
				header_urll=u"没有头像链接:"
			print u"头像链接:"+header_urll
			image_url=content.xpath('//div[@class="small_pic_box fn-clear"]//div[@class="small_pic fn-clear"]//li//img//@src')
			print u"相册链接:",
			print image_url
			content1=content.xpath('//div[@class="main_1000 mt15 fn-clear"]//div[@class="bg_white"]//div[@class="js_text"]//text()')
			content2=""
			for i in content1:
				content2+=i
			content3=content2
			print u"内心独白:"+content3.strip()
			place=content.xpath('//div[@class="main_1000 bg_white mt15"]//h6[@class="member_name"]/a[2]/text()')
			if len(place)==1:
				where=place[0]
			else:
				where=u"河南"
			print u"来自:"+where+u"省"
			xueli=content.xpath('//div[@class="main_1000 bg_white mt15"]//ul[@class="member_info_list fn-clear"]/li[1]//div[@class="fl pr"]/em/text()')
			if len(xueli)==1:
				print u"学历:"+xueli[0]
			else:
				print u"学历:本科"
			print "***********************************************"
		if self.page<=5:
			self.meiyiye()
if __name__=="__main__":
	ni=we()
	ni.meiyiye()