import requests
from lxml import etree
import multiprocessing
def get_all_proxy(queue):
url = 'http://www.xicidaili.com/nn/1'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
response = requests.get(url,headers=headers)
html = response.text
html = etree.HTML(html)
ip_list = html.xpath('//table[@id="ip_list"]/tr/td[2]/text()')
d_list = html.xpath('//table[@id="ip_list"]/tr/td[3]/text()')
for i in range(0,len(ip_list)):
proxy = 'http://' + ip_list[i] + ':' + d_list[i]
queue.put(proxy)
def check_all_proxy(proxy):
try:
url = 'http://fanyi.baidu.com/sug'
porxy_dict = {
'http':proxy
}
try:
response = requests.get(url,proxies=porxy_dict,timeout=3)
if response.status_code == 200:
print(proxy)
return proxy
else:
print('不可用'*20)
return proxy
except:
return None
except Exception as e:
print(e)
if __name__ == '__main__':
import time
start_time = time.time()
q = multiprocessing.Queue()
p = multiprocessing.Process(target=get_all_proxy, args=(q,))
p.start()
pool = multiprocessing.Pool(50)
result_list = []
while True:
try:
proxy_str = q.get(timeout=5)
except:
break
proxy_res = pool.apply_async(check_all_proxy, (proxy_str,))
result_list.append(proxy_res)
valid_proxy_list = []
for proxy_res in result_list:
result = proxy_res.get()
if result is None:
pass
else:
valid_proxy_list.append(result)
print('All proxy we can get:')
print(valid_proxy_list)
pool.close()
pool.join()
p.join()