如何爬取微博全部粉丝python_让你了解python的强大！【 登录新浪微博爬取粉丝信息】...

环境：

系统：windows 7

版本：python 3.3

IDE: PyCharm 4.0.4

#!/usr/bin/env python3

# -*- coding: utf-8 -*-

import time

import base64

import rsa

import binascii

import requests

import re

import random

try:

import cookielib

except:

import http.cookiejar as cookielib

try:

from PIL import Image

except:

pass

try:

from urllib.parse import quote_plus

except:

from urllib import quote_plus

'''

如果没有开启登录保护，不用输入验证码就可以登录

如果开启登录保护，需要输入验证码

'''

# 构造 Request headers

agent = 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0'

headers = {

# "Host": "www.weibo.com",

'User-Agent': agent

}

session = requests.session()

session.cookies = cookielib.LWPCookieJar(filename='cookies')

try:

session.cookies.load(ignore_discard=True)

except:

print("Cookie 未能加载")

# 访问 初始页面带上 cookie

index_url = "http://weibo.com/login.php"

try:

session.get(index_url, headers=headers, timeout=2)

except:

session.get(index_url, headers=headers)

try:

input = raw_input

except:

pass

def get_su(username):

"""

对 email 地址和手机号码 先 javascript 中 encodeURIComponent

对应 Python 3 中的是 urllib.parse.quote_plus

然后在 base64 加密后decode

"""

username_quote = quote_plus(username)

username_base64 = base64.b64encode(username_quote.encode("utf-8"))

return username_base64.decode("utf-8")

# 预登陆获得 servertime, nonce, pubkey, rsakv

def get_server_data(su):

pre_url = "http://login.sina.com.cn/sso/prelogin.php?entry=weibo&callback=sinaSSOController.preloginCallBack&su="

pre_url = pre_url + su + "&rsakt=mod&checkpin=1&client=ssologin.js(v1.4.18)&_="

pre_url = pre_url + str(int(time.time() * 1000))

pre_data_res = session.get(pre_url, headers=headers)

sever_data = eval(pre_data_res.content.decode("utf-8").replace("sinaSSOController.preloginCallBack", ''))

return sever_data

# print(sever_data)

def get_password(password, servertime, nonce, pubkey):

rsaPublickey = int(pubkey, 16)

key = rsa.PublicKey(rsaPublickey, 65537)  # 创建公钥

message = str(servertime) + '\t' + str(nonce) + '\n' + str(password)  # 拼接明文js加密文件中得到

message = message.encode("utf-8")

passwd = rsa.encrypt(message, key)  # 加密

passwd = binascii.b2a_hex(passwd)  # 将加密信息转换为16进制。

return passwd

def get_cha(pcid):

cha_url = "http://login.sina.com.cn/cgi/pin.php?r="

cha_url = cha_url + str(int(random.random() * 100000000)) + "&s=0&p="

cha_url = cha_url + pcid

cha_page = session.get(cha_url, headers=headers)

with open("cha.jpg", 'wb') as f:

f.write(cha_page.content)

f.close()

try:

im = Image.open("cha.jpg")

im.show()

im.close()

except:

print(u"请到当前目录下，找到验证码后输入")

def login(username, password):

# su 是加密后的用户名

su = get_su(username)

sever_data = get_server_data(su)

servertime = sever_data["servertime"]

nonce = sever_data['nonce']

rsakv = sever_data["rsakv"]

pubkey = sever_data["pubkey"]

showpin = sever_data["showpin"]

password_secret = get_password(password, servertime, nonce, pubkey)

postdata = {

'entry': 'weibo',

'gateway': '1',

'from': '',

'savestate': '7',

'useticket': '1',

'pagerefer': "http://login.sina.com.cn/sso/logout.php?entry=miniblog&r=http%3A%2F%2Fweibo.com%2Flogout.php%3Fbackurl",

'vsnf': '1',

'su': su,

'service': 'miniblog',

'servertime': servertime,

'nonce': nonce,

'pwencode': 'rsa2',

'rsakv': rsakv,

'sp': password_secret,

'sr': '1366*768',

'encoding': 'UTF-8',

'prelt': '115',

'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',

'returntype': 'META'

}

login_url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.18)'

if showpin == 0:

login_page = session.post(login_url, data=postdata, headers=headers)

else:

pcid = sever_data["pcid"]

get_cha(pcid)

postdata['door'] = input(u"请输入验证码")

login_page = session.post(login_url, data=postdata, headers=headers)

login_loop = (login_page.content.decode("GBK"))

# print(login_loop)

pa = r'location\.replace\([\'"]http://blog.csdn.net/dai_jing/article/details/(.*?)[\'"]\)'

loop_url = re.findall(pa, login_loop)[0]

# print(loop_url)

# 此出还可以加上一个是否登录成功的判断，下次改进的时候写上

login_index = session.get(loop_url, headers=headers)

uuid = login_index.text

uuid_pa = r'"uniqueid":"http://blog.csdn.net/dai_jing/article/details/(.*?)"'

uuid_res = re.findall(uuid_pa, uuid, re.S)[0]

web_weibo_url = "http://weibo.com/%s/profile?topnav=1&wvr=6&is_all=1" % uuid_res

weibo_page = session.get(web_weibo_url, headers=headers)

weibo_pa = r'

上面引用的lib3是我自己写的一个http接口

fileWriter.py

spider.py

效果图：

注意事项：

1，新浪微博本身有查看限制，只能查看到前5页的数据

2，爬虫工具可以使用火狐浏览器，装上fireDebug 插件用于拦截消息，获取cookie和其他参数

python学习资料交流群：547464842  每天会更新视频资料，随时可以下载。