使用代理爬去微信公众号_爬虫之使用代理爬取微信公众号文章（下）

一 修改代理池

1 将代理池中TEST_URL的地址修改为：TEST_URL = 'https://weixin.sogou.com/weixin?type=2&query=NBA'

2 运行代理池程序，运行结果如下：

这样数据库中留下100分的代理就是针对搜狗微信的可用代理了。

同时访问代理接口：http://127.0.0.1:5555/random

返回结果如下：

二 爬虫代码

from requests import Session

from weixin.config import *

from weixin.db import RedisQueue

from weixin.mysql import MySQL

from weixin.request import WeixinRequest

from urllib.parse import urlencode

import requests

from pyquery import PyQuery as pq

from requests import ReadTimeout, ConnectionError

class Spider():

base_url = 'http://weixin.sogou.com/weixin'

keyword = 'NBA'

headers = {

'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

'Accept-Encoding': 'gzip, deflate',

'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6,ja;q=0.4,zh-TW;q=0.2,mt;q=0.2',

'Cache-Control': 'max-age=0',

'Connection': 'keep-alive',

'Cookie': 'IPLOC=CN1100; SUID=6FEDCF3C541C940A000000005968CF55; SUV=1500041046435211; ABTEST=0|1500041048|v1; SNUID=CEA85AE02A2F7E6EAFF9C1FE2ABEBE6F; weixinIndexVisited=1; JSESSIONID=aaar_m7LEIW-jg_gikPZv; ld=Wkllllllll2BzGMVlllllVOo8cUlllll5G@HbZllll9lllllRklll5@@@@@@@@@@; LSTMV=212%2C350; LCLKINT=4650; ppinf=5|1500042908|1501252508|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo1NDolRTUlQjQlOTQlRTUlQkElODYlRTYlODklOEQlRTQlQjglQTglRTklOUQlOTklRTglQTclODV8Y3J0OjEwOjE1MDAwNDI5MDh8cmVmbmljazo1NDolRTUlQjQlOTQlRTUlQkElODYlRTYlODklOEQlRTQlQjglQTglRTklOUQlOTklRTglQTclODV8dXNlcmlkOjQ0Om85dDJsdUJfZWVYOGRqSjRKN0xhNlBta0RJODRAd2VpeGluLnNvaHUuY29tfA; pprdig=ppyIobo4mP_ZElYXXmRTeo2q9iFgeoQ87PshihQfB2nvgsCz4FdOf-kirUuntLHKTQbgRuXdwQWT6qW-CY_ax5VDgDEdeZR7I2eIDprve43ou5ZvR0tDBlqrPNJvC0yGhQ2dZI3RqOQ3y1VialHsFnmTiHTv7TWxjliTSZJI_Bc; sgid=27-27790591-AVlo1pzPiad6EVQdGDbmwnvM; PHPSESSID=mkp3erf0uqe9ugjg8os7v1e957; SUIR=CEA85AE02A2F7E6EAFF9C1FE2ABEBE6F; sct=11; ppmdig=1500046378000000b7527c423df68abb627d67a0666fdcee; successCount=1|Fri, 14 Jul 2017 15:38:07 GMT',

'Host': 'weixin.sogou.com',

'Upgrade-Insecure-Requests': '1',

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'

}

session = Session()

queue = RedisQueue()

mysql = MySQL()

def get_proxy(self):

"""

从代理池获取代理

:return:

"""

try:

response = requests.get(PROXY_POOL_URL)

if response.status_code == 200:

print('Get Proxy', response.text)

return response.text

return None

except requests.ConnectionError:

return None

def start(self):

"""

初始化工作

"""

# 全局更新Headers

self.session.headers.update(self.headers)

start_url = self.base_url + '?' + urlencode({'query': self.keyword, 'type': 2})

weixin_request = WeixinRequest(url=start_url, callback=self.parse_index, need_proxy=True)

# 调度第一个请求

self.queue.add(weixin_request)

def parse_index(self, response):

"""

解析索引页

:param response: 响应

:return: 新的响应

"""

doc = pq(response.text)

items = doc('.news-box .news-list li .txt-box h3 a').items()

for item in items:

url = item.attr('href')

weixin_request = WeixinRequest(url=url, callback=self.parse_detail)

yield weixin_request

next = doc('#sogou_next').attr('href')

if next:

url = self.base_url + str(next)

weixin_request = WeixinRequest(url=url, callback=self.parse_index, need_proxy=True)

yield weixin_request

def parse_detail(self, response):

"""

解析详情页

:param response: 响应

:return: 微信公众号文章

"""

doc = pq(response.text)

data = {

'title': doc('.rich_media_title').text(),

'content': doc('.rich_media_content').text(),

'nickname': doc('#js_profile_qrcode > div > strong').text(),

'wechat': doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()

}

yield data

def request(self, weixin_request):

"""

执行请求

:param weixin_request: 请求

:return: 响应

"""

try:

if weixin_request.need_proxy:

proxy = self.get_proxy()

if proxy:

proxies = {

'http': 'http://' + proxy,

'https': 'https://' + proxy

}

return self.session.send(weixin_request.prepare(),

timeout=weixin_request.timeout, allow_redirects=False, proxies=proxies)

return self.session.send(weixin_request.prepare(), timeout=weixin_request.timeout, allow_redirects=False)

except (ConnectionError, ReadTimeout) as e:

print(e.args)

return False

def error(self, weixin_request):

"""

错误处理

:param weixin_request: 请求

:return:

"""

weixin_request.fail_time = weixin_request.fail_time + 1

print('Request Failed', weixin_request.fail_time, 'Times', weixin_request.url)

if weixin_request.fail_time < MAX_FAILED_TIME:

self.queue.add(weixin_request)

def schedule(self):

"""

调度请求

:return:

"""

while not self.queue.empty():

weixin_request = self.queue.pop()

callback = weixin_request.callback

print('Schedule', weixin_request.url)

response = self.request(weixin_request)

if response and response.status_code in VALID_STATUSES:

results = list(callback(response))

if results:

for result in results:

print('New Result', type(result))

if isinstance(result, WeixinRequest):

self.queue.add(result)

if isinstance(result, dict):

self.mysql.insert('articles', result)

else:

self.error(weixin_request)

else:

self.error(weixin_request)

def run(self):

"""

入口

:return:

"""

self.start()

self.schedule()

if __name__ == '__main__':

spider = Spider()

spider.run()

三 数据库

1 程序

import pymysql

from weixin.config import *

class MySQL():

def __init__(self, host=MYSQL_HOST, username=MYSQL_USER, password=MYSQL_PASSWORD, port=MYSQL_PORT,

database=MYSQL_DATABASE):

"""

MySQL初始化

:param host:

:param username:

:param password:

:param port:

:param database:

"""

try:

self.db = pymysql.connect(host, username, password, database, charset='utf8', port=port)

self.cursor = self.db.cursor()

except pymysql.MySQLError as e:

print(e.args)

def insert(self, table, data):

"""

插入数据

:param table:

:param data:

:return:

"""

keys = ', '.join(data.keys())

values = ', '.join(['%s'] * len(data))

sql_query = 'insert into %s (%s) values (%s)' % (table, keys, values)

try:

self.cursor.execute(sql_query, tuple(data.values()))

self.db.commit()

except pymysql.MySQLError as e:

print(e.args)

self.db.rollback()

2 数据表脚本

CREATE TABLE `articles` (

`id` int(11) NOT NULL AUTO_INCREMENT,

`title` varchar(255) DEFAULT NULL,

`content` text,

`wechat` varchar(255) DEFAULT NULL,

`nickname` varchar(255) DEFAULT NULL,

PRIMARY KEY (`id`)

) ENGINE=InnoDB AUTO_INCREMENT=151 DEFAULT CHARSET=utf8;