协程大批量爬取是要被封IP的，最优秀的方法就是在被封IP时候立马切换IP

一. 多协程爬虫遇到的难点

    之前本着一封IP就切换IP的原则做了个协程爬虫。但是操作并发运行的爬虫和单线程的爬虫的难度真的是云泥之别。因为是并发运行的爬虫，用的IP是同一个，被封的时候当然是全部IP一起封了。

    而执行操作的时候，又会每个协程换一次，这就会导致IP的浪费和爬虫运行的缓慢。

二. 解决方案

    并发问题，自然要用到协程间通讯，Event。具体思想就是，当一个协程被封IP了，在切换IP的方法中，只让一条协程运行，其他全部休眠，待到IP切换成功再唤醒其余的协程。

为了方便调试，做了个socket服务端和客户端来模拟。

服务端：

#-*- coding:utf-8 -*-
from gevent import monkey; monkey.patch_all()

import socket
import json
import gevent
import threading

s = socket.socket()
host = '127.0.0.1'
port = 9991
s.bind((host, port))
ban_ip = []
ips = []

def processSocket(c):
	msg = c.recv(1024)
	json_msg = json.loads(msg.decode('utf-8').replace('\'', '"'))

	if json_msg['url'] == 'break':
		print('接收到消息， 退出')
		c.send('已成功退出，谢谢!!'.encode('utf-8'))
		c.close()
		s.close()
	else:
		if json_msg['ip'] in ban_ip:
			print(json_msg['ip'])
			print('恶意IP，查封！！')
			c.send('304'.encode('utf-8'))
			c.close()
		else:
			ips.append(json_msg['ip'])
			if len(ips) == 5:
				an = set(ips)
				if len(list(an)) ==	1:
					ban_ip.append(an.pop())
					ips.clear()
					if len(ips) == 0:
						print('已经清空...')
			print('接收到的信息：%s' %(msg.decode('utf-8')))
			c.send('200'.encode('utf-8'))
			c.close()	

print('socket 服务器开启!!!')
s.listen(50)
while True:
	c, addr = s.accept()
	gevent.spawn(processSocket, c)

客户端：

#-*- coding:utf-8 -*-
from gevent.pool import Pool
from gevent.lock import BoundedSemaphore
from gevent.event import Event
from gevent import monkey; monkey.patch_all()
# from threading import Event
from time import sleep

import socket
import random
import gevent
import threading





class client(object):

	def __init__(self):
		self.host = '127.0.0.1'
		self.port = 9991
		self.ip = 1
		self.isChanging = False
		self.event = Event()
		self.l = [{'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'},
			 {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'},
			 {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'a'}, {'ip':self.ip, 'url':'break'}]	

	def sendMsg(self, msg):
		# sleep(random.random()*5)
		s = socket.socket()
		gevent.sleep(random.random()*5)
		s.connect((self.host, self.port))
		msg['ip'] = self.ip
		s.send(str(msg).encode('utf-8'))

		resp = s.recv(1024)
		if resp.decode('utf-8') == '200':
			print('连接通过!!')
		elif resp.decode('utf-8') == '304':
			if self.isChanging:
				print('IP切换中， 休眠...')
				# event.wait()
				self.event.wait()
				msg['ip']=self.ip
				self.sendMsg(msg)
			else:
				print('IP被ban..请切换')
				self.checkoutIP()
		else:
			print(resp.decode('utf-8'))


	def checkoutIP(self):
		self.isChanging = True
		self.event.clear()
		self.ip = self.ip+1

		print('IP切换中...')
		gevent.sleep(5)
		# Event.set()
		# sleep(random.random()*5)
		print('切换成：%s' %(self.ip))
		self.isChanging = False
		self.event.set()

	def multiprocess(self):
		# for i in l:
		# 	t = threading.Thread(target=sendMsg, args=(i,)) 
		# 	t.start()
		p = Pool(3)
		threads = [p.spawn(self.sendMsg, s) for s in self.l]
		gevent.joinall(threads)


if __name__ == "__main__":
	c = client()
	# c.sendMsg({"ip":3, "url":"break"})
	c.multiprocess()

运行实况：