import requests
from queue import Queue
import random
import gevent
from gevent import monkey
monkey.patch_all()
def getheaders():
user_agent_list = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"
]
UserAgent = random.choice(user_agent_list)
headers = {"User-Agent": UserAgent}
return headers
def is_enable(ip_port, q):
proxies = {
"http": "http://" + ip_port,
"https": "http://" + ip_port,
}
try:
res = requests.get('https://www.baidu.com/', headers=getheaders(),
proxies=proxies, timeout=2)
q.put(ip_port+' 能用')
# print(ip_port, '能用')
except Exception as e:
q.put(ip_port+' 不能用')
# print(ip_port, '不能用')
def main():
ips_ports_list = []
# ips.txt: 50个从西刺爬取的IP
with open('ips.txt') as f:
for line in f:
ips_ports_list.append(line.strip())
q = Queue()
job_list = [gevent.spawn(is_enable, ip_port, q) for ip_port in ips_ports_list]
# gevent.joinall(job_list)
completed_num = 0
while True:
output = q.get()
print(output)
completed_num += 1
if completed_num >= len(ips_ports_list):
break
if __name__ == '__main__':
main()
Python协程gevent测试IP是否可用
最新推荐文章于 2022-12-06 15:20:44 发布