新浪微博爬虫模拟登录（爬手机版）

新浪微博爬虫模拟登录

最近一直在想做新浪微博爬取，开始的时候做了一个PC版的爬取，但是发现提取内容真心难过所以在这里爬了手机版的话不多说，show you my code 这个可以提取固定人微博内容，接下来的工作是要放入到mysql提取大量内容做一些聚类分析，这个验证码是要自己输入的，等我把验证码自动识别那部分做好后再来补充，下面是微博内容爬取

#-*-coding:utf8-*-

import requests
from lxml import etree
import urllib
import cookielib
import urllib2


s = requests.Session()#记住cookies

url = 'http://weibo.cn/u/1906257933' #此处请修改为微博地址
html_you = s.get(url).content
selector_you = etree.HTML(html_you)
url_login = selector_you.xpath('//a[@id="top"]/@href')[0]

html = s.get(url_login).content
selector = etree.HTML(html)
password = selector.xpath('//input[@type="password"]/@name')[0]
vk = selector.xpath('//input[@name="vk"]/@value')[0]
action = selector.xpath('//form[@method="post"]/@action')[0]
capId = selector.xpath('//input[@name="capId"]/@value')[0]
url_1 = 'http://weibo.cn/interface/f/ttt/captcha/show.php?cpt='+capId
path = "d://downloads//1.GIF"
data = urllib.urlretrieve(url_1,path)
print'Pic Saved!'
print capId
print action
print password
print vk
code = raw_input('please input the:')
new_url = url_login + action
print new_url
data = {
    'backTitle' : u'手机新浪网',
    'backURL' : 'http://weibo.cn/u/1906257933', #此处请填写微博地址
    'capId' : capId,
    'code': code,
    'mobile' : '登录名',
     password : '密码',
    'remember' : 'on',
    'tryCount' : '',
    'vk' : vk,
    'submit' : u'登录'
    }

newhtml = s.post(new_url,data=data).content
new_selector = etree.HTML(newhtml, parser=etree.HTMLParser(encoding='UTF-8'))
page = new_selector.xpath('//input[@type="hidden"]/@value')[0]
print page
for i in range(1,int(page)+1):
    url_page = 'http://weibo.cn/u/1906257933?page=%s'%i
    url_page_1 = s.get(url_page).content
    new_selector_1 = etree.HTML(url_page_1, parser=etree.HTMLParser(encoding='UTF-8'))
    content = new_selector_1.xpath('//span[@class="ctt"]')
    for each in content:
        text = each.xpath('string(.)')
        print text