本文目的：介绍如何抓取微博内容，利用requests包+cookies实现登陆微博，lxml包的xpath语法解析网页，抓取目标内容。
所需python包：requests、lxml 皆使用pip安装即可
XPath即为XML路径语言，它是一种用来确定XML（标准通用标记语言的子集）文档中某部分位置的语言。XPath基于XML的树状结构，提供在数据结构树中找寻节点的能力。 xPath 同样也支持HTML.
XPath 是一门小型的查询语言，这里我们将它与 python 爬虫相结合来介绍。
xpath使用方法简介：
step1: 安装 lxml 库。
step2: from lxml import etree
step3: Selector = etree.HTML(网页源代码)
step4: Selector.xpath(一段神奇的符号)
具体的xpath解析方法大家可以参考之后下边的代码或者参考这篇博文http://cighao.com/2016/03/01/introduction-of-xPath/
言归正传，接下来从头看一下抓取的思路，首先找到微博的网址，为了方便抓取，准备抓取3g版微博，网址为 https://weibo.cn/，进而找到登陆的网址为 https://passport.weibo.cn/signin/login，按F12调出开发者工具，点击最上边一栏的network，然后输入自己的账号密码进行登录，登录之后可以看到弹出了很多元素，找到m.weibo.cn这一项点击，如图所示，里面三个红色框就是我们需要的
Request URL 代表了我们刚刚请求的网址
cookies代表了刚刚登录我们的账号里面包含的密码信息，待会登录会用到
user-Agent代表了我们使用的浏览器的头部信息，用来对我们的爬虫进行伪装，降低被封禁的风险
接下来上一段代码
#coding=utf-8 import re import time import string import os import pickleimport requests from lxml import etree import traceback import sys reload(sys) sys.setdefaultencoding('utf-8') class weibo: cookie = {"Cookie": "~~~~~~~~~~"} #将your cookie替换成自己的cookie header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'}#这里就是浏览器的头部信息 def __init__(self,user_id,filter = 0): self.user_id = user_id #用户id，即需要我们输入的数字，如昵称为“Dear-迪丽热巴”的id为1669879400 self.filter = filter #取值范围为0、1，程序默认值为0，代表要爬取用户的全部微博，1代表只爬取用户的原创微博 self.userName = '' #用户名，如“Dear-迪丽热巴” self.weiboNum = 0 #用户全部微博数 self.weiboNum2 = 0 #爬取到的微博数 self.following = 0 #用户关注数 self.followers = 0 #用户粉丝数 self.weibos = [] #微博内容 self.num_zan = [] #微博对应的点赞数 self.num_forwarding = [] #微博对应的转发数 self.num_comment = [] #微博对应的评论数 self.weibo_detail_urls=[]#pickle.load(open("weibourl1.pkl", "r"))#微博评论 self.weibourls=[]#pickle.load(open('weibourl2.pkl','r'))#每一条微博链接，用于断点续爬 def getUserName(self): try: url = 'http://weibo.cn/%d/info'%(self.user_id) html = requests.get(url, cookies = weibo.cookie,headers=weibo.header).content #这一句里面的cookies与headers就实现了登录 selector = etree.HTML(html) userName = selector.xpath("//title/text()")[0] self.userName = userName[:-3] #print '用户昵称：' + self.userName except Exceptione,e: print "Error: ",e traceback.print_exc()
上边的代码就实现了利用requests包+cookies登录微博，接下来就是找到想要抓取的用户id，然后使用xpath解析网页进行信息抓取，代码如下：
def getUserInfo(self): try: url = 'http://weibo.cn/u/%d?filter=%d&page=1'%(self.user_id,self.filter) html = requests.get(url, cookies = weibo.cookie,headers=weibo.header).content selector = etree.HTML(html) pattern = r"\d+\.?\d*" str_wb = selector.xpath("//div[@class='tip2']/span[@class='tc']/text()") guid = re.findall(pattern, str_wb[0], re.S|re.M) for value in guid: num_wb = int(value) break self.weiboNum = num_wb print '微博数: ' + str(self.weiboNum) str_gz = selector.xpath("//div[@class='tip2']/a/text()")[0] guid = re.findall(pattern, str_gz, re.M) self.following = int(guid[0]) print '关注数: ' + str(self.following) str_fs = selector.xpath("//div[@class='tip2']/a/text()")[1] guid = re.findall(pattern, str_fs, re.M) self.followers = int(guid[0]) print '粉丝数: ' + str(self.followers) except Exception,e: print "Error: ",e traceback.print_exc() def getWeiboInfo(self): try: url = 'http://weibo.cn/u/%d?filter=%d&page=1'%(self.user_id,self.filter) html = requests.get(url, cookies = weibo.cookie,headers=weibo.header).content selector = etree.HTML(html) if selector.xpath('//input[@name="mp"]')==[]: pageNum = 1 else: pageNum = (int)(selector.xpath('//input[@name="mp"]')[0].attrib['value']) pattern = r"\d+\.?\d*" f = open("./%s.txt"%self.user_id, "wb") for page in range(1,pageNum+1): if page % 10 == 0: print('[ATTEMPTING] rest for 5 minutes to cheat weibo site, avoid being banned.') time.sleep(60*5) url2 = 'http://weibo.cn/u/%d?filter=%d&page=%d'%(self.user_id,self.filter,page) html2 = requests.get(url2, cookies = weibo.cookie,headers=weibo.header).content selector2 = etree.HTML(html2) info = selector2.xpath("//div[@class='c']") if len(info) > 3: for i in range(0,len(info)-2): detail = info[i].xpath("@id")[0] url3='http://weibo.cn/comment/{}?uid={}&rl=0'.format(detail.split('_')[-1], self.user_id) if url3 not in self.weibo_detail_urls: self.weiboNum2 = self.weiboNum2 + 1 #print self.weibo_detail_urls str_t = info[i].xpath("div/span[@class='ctt']") weibos = str_t[0].xpath('string(.)') self.weibos.append(weibos) #print '微博内容：'+ weibos+'***'+'No.%s'%self.weiboNum2 str_zan = info[i].xpath("div/a/text()")[-4] guid = re.findall(pattern, str_zan, re.M) num_zan = int(guid[0]) self.num_zan.append(num_zan) #print '点赞数: ' + str(num_zan) forwarding = info[i].xpath("div/a/text()")[-3] guid = re.findall(pattern, forwarding, re.M) num_forwarding = int(guid[0]) self.num_forwarding.append(num_forwarding) #print '转发数: ' + str(num_forwarding) comment = info[i].xpath("div/a/text()")[-2] guid = re.findall(pattern, comment, re.M) num_comment = int(guid[0]) self.num_comment.append(num_comment) #print '评论数: ' + str(num_comment) self.weibo_detail_urls.append(url3) text=str(self.weiboNum2)+':'+weibos + '\t'+'点赞数：' + str(num_zan) + '\t'+ ' 转发数：' + str(num_forwarding) +'\t'+ ' 评论数：' + str(num_comment) + '\n' f.write(text) pickle.dump(self.weibo_detail_urls, open("weibourl1.pkl", "w")) else: print url3+'这条微博已经爬取过，忽略' if self.filter == 0: print '共'+str(self.weiboNum2)+'条微博' else: print '共'+str(self.weiboNum)+'条微博，其中'+str(self.weiboNum2)+'条为原创微博' except Exception,e: print "Error: ",e traceback.print_exc() def get_weibo_detail_comment(self): weibo_comments_save_path = './weibo/{}.txt'.format(self.user_id) with open(weibo_comments_save_path, 'a') as f: for i, url in enumerate(self.weibo_detail_urls): if url not in self.weibourls: self.weibourls.append(url) pickle.dump(self.weibourls, open("weibourl2.pkl", "w")) print('solving weibo detail from {}'.format(url)) html_detail = requests.get(url, cookies=weibo.cookie, headers=weibo.header).content selector = etree.HTML(html_detail) str1='id="pagelist"' if str1 in html_detail: all_comment_pages = selector.xpath('//*[@id="pagelist"]/form/div/input[1]/@value')[0] else: all_comment_pages = 1 print('\n这是 {} 的微博：'.format(self.userName)) #print('微博内容： {}'.format(self.weibos[i])) #print('接下来是下面的评论：\n\n') f.writelines('微博内容： {}'.format(self.weibos[i])+'\n') f.writelines('接下来是下面的评论:\n') for page in range(1,int(all_comment_pages) + 1): if page % 10 == 0: print('[ATTEMPTING] rest for 5 minutes to cheat weibo site, avoid being banned.') time.sleep(60*5) detail_comment_url = url + '&page=' + str(page) try: html_detail_page = requests.get(detail_comment_url, cookies=weibo.cookie,headers=weibo.header).content selector = etree.HTML(html_detail_page) comment_div_element = selector.xpath('//div[starts-with(@id, "C_")]') for child in comment_div_element: single_comment_user_name = child.xpath('a[1]/text()')[0] if child.xpath('span[1][count(*)=0]'): single_comment_content = child.xpath('span[1][count(*)=0]/text()')[0] else: span_element = child.xpath('span[1]')[0] at_user_name = span_element.xpath('a/text()')[0] at_user_name = '$' + at_user_name.split('@')[-1] + '$' single_comment_content = span_element.xpath('/text()') single_comment_content.insert(1, at_user_name) single_comment_content = ' '.join(single_comment_content) full_single_comment = '<' + single_comment_user_name + '>' + ': ' + single_comment_content #print(full_single_comment) f.writelines(full_single_comment + '\n') #f.writelines('F\n') except etree.XMLSyntaxError as e: print('user id {} all done!'.format(self.user_id)) print('all weibo content and comments saved into {}'.format(weibo_comments_save_path)) f.writelines('F\n') else: print 'has alredy' def writeTxt(self): try: if self.filter == 1: resultHeader = '\n\n原创微博内容：\n' else: resultHeader = '\n\n微博内容：\n' result = '用户信息\n用户昵称：' + self.userName + '\n用户id：' + str(self.user_id) + '\n微博数：' + str(self.weiboNum) + '\n关注数：' + str(self.following) + '\n粉丝数：' + str(self.followers) + resultHeader if os.path.isdir('weibo') == False: os.mkdir('weibo') f = open("./%s.txt"%self.user_id, "wb") f.write(result) f.close() except Exception,e: print "Error: ",e traceback.print_exc() def start(self): try: weibo.getUserName(self) weibo.getUserInfo(self) weibo.writeTxt(self) weibo.getWeiboInfo(self) weibo.get_weibo_detail_comment(self) print '信息抓取完毕' print '===========================================================================' except Exception,e: print "Error: ",e user_id =~~~~~~~~~~~~~ #可以改成任意合法的用户id（爬虫的微博id除外） filter = 0 #值为0表示爬取全部的微博信息（原创微博+转发微博），值为1表示只爬取原创微博 #open('./weibourl1.pkl','w') #open('./weibourl2.pkl','w') wb = weibo(user_id,filter) #调用weibo类，创建微博实例wb wb.start() #爬取微博信息 print '用户名：' + wb.userName print '全部微博数：' + str(wb.weiboNum) print '关注数：' + str(wb.following) print '粉丝数：' + str(wb.followers) print '最新一条微博为：' + wb.weibos[0] #若filter=1则为最新的原创微博，如果该用户微博数为0，即len(wb.weibos)==0,打印会出错，下同 print '最新一条微博获得的点赞数：' + str(wb.num_zan[0]) print '最新一条微博获得的转发数：' + str(wb.num_forwarding[0]) print '最新一条微博获得的评论数：' + str(wb.num_comment[0]) wb.writeTxt() #wb.writeTxt()只是把信息写到文件里，大家可以根据自己的需要重新编写writeTxt()函数
这两段代码就实现了对微博用户的用户名、微博数、微博转发、评论、点赞数、以及微博及对微博评论的抓取。
参考博文：http://cighao.com/2016/03/01/introduction-of-xPath/
5236
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
