每一个代理源单独异步去爬不方便,所以将所有的代理源的url集中起来,添加到任务队列里,爬出的结果再去根据代理源解析ip(大部分都只取前几页,后面的基本失效了)
将target_url中'http://icanhazip.com'改为'https://icanhazip.com'就可以检验IP是否支持https了(成功率大概只有1%-5%,免费的要什么自行车...哈哈哈)
要添加新的代理源也可像我这样去添加(不建议),爬出来的结果建议单独封装去取ip(我这样瞎搞迟早废掉)
import re
import asyncio
import json
import aiohttp
from keywords import Get
from socketbase import Sendmethod
class Proxy:
def __init__(self):
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
self.url_list=[]
self.proixes={}
self.good_proixes={}
self.index=0
self.target_url='http://icanhazip.com'
def kuaidaili(self):
url='https://www.kuaidaili.com/free/inha/'
for i in range(1,3):
self.url_list.append(('kuaidaili',url+str(i)+'/'))
def xiaohuan(self):
url='https://ip.ihuan.me/address/5Lit5Zu9.html?page='
pages=['b97827cc','4ce63706','5crfe930']
for page in pages:
self.url_list.append(('xiaohuan',url + page))
def ip66(self):
url = 'http://www.66ip.cn/index.html'
self.url_list.append(('ip66',url))
def dieniao(self):
url='https://www.dieniao.com/FreeProxy.html'
self.url_list.append(('dieniao',url))
def kaixin(self):
url='https://proxy11.com/api/demoweb/proxy.json'
self.url_list.append(('kaixin',url))
def yundaili(self):
url='http://www.ip3366.net/?stype=1&page='
for i in range(1,4):
self.url_list.append(('yundaili',url+str(i)))
def freedaili(self):
for i in range(1,4):
url=f'https://ip.jiangxianli.com/?page={i}&country=%E4%B8%AD%E5%9B%BD'
self.url_list.append(('freedaili', url))
def daili89(self):
for i in range(1, 7):
url=f'https://www.89ip.cn/index_{i}.html'
self.url_list.append(('daili89', url))
def zhandaye(self):
main_url='https://www.zdaye.com'
self.header={
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0'
}
res = Sendmethod().method('get', main_url+'/dayProxy.html',headers=self.header)
main_pages = re.findall('<H3 class="thread_title"><a href="(.*?)">', res.text)
for main_page in main_pages[0:3]:
url=main_url+main_page
for x in range(6):
if x ==0:
self.url_list.append(('zhandaye', url))
else:
self.url_list.append(('zhandaye', url[:-5]+'/'+str(x)+'.html'))
async def crawl_ip(self,item):
timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession(trust_env=True) as conn:
async with conn.get(item[1], timeout=timeout, ssl=False) as res:
response = await res.text()
if item[0] == 'kuaidaili':
ips = re.findall('<td data-title="IP">(.*?)</td>', response)
ports = re.findall('<td data-title="PORT">(.*?)</td>', response)
for ip, port in zip(ips, ports):
self.proixes[ip] = port
elif item[0] == 'xiaohuan':
content = re.findall('<img src="/flag/CN.svg">(.*?) href="/address/5Lit5Zu9.html">', response)
for i in content:
ip = re.findall('(.*?)</a></td><td>', i)
port = re.findall('</a></td><td>(.*?)</td><td><a', i)
self.proixes[ip[0]] = port[0]
elif item[0] == 'ip66':
ips = re.findall('<tr><td>(.*?)</td><td>', response)[1:]
ports = re.findall('</td><td>(.*?)</td><td>', response)
ports = list(filter(lambda x: x.isdigit(), ports))
for ip, port in zip(ips, ports):
self.proixes[ip] = port
elif item[0] == 'dieniao':
ips = re.findall("<span class='f-address'>(.*?)</span>",response)[1:]
ports = re.findall("<span class='f-port'>(.*?)</span>", response)
ports = list(filter(lambda x: x.isdigit(), ports))
for ip, port in zip(ips, ports):
self.proixes[ip] = port
elif item[0] == 'kaixin':
ips = Get.get_keyword(json.loads(response), 'ip')
ports = Get.get_keyword(json.loads(response), 'port')
for ip, port in zip(ips, ports):
self.proixes[ip] = port
elif item[0] == 'yundaili':
content = re.findall("<tr>(.*?)</tr>", response, re.S)[1:]
for item in content:
ip = re.findall("<td>(.*?)</td>", item)[0]
port = re.findall("<td>(.*?)</td>", item)[1]
self.proixes[ip] = port
elif item[0] == 'freedaili':
content = re.findall('<link rel="dns-prefetch" href="//github.com">(.*?)<link rel="dns-prefetch" href="//buy.jiangxianli.com">',response, re.S)
ips = re.findall('<link rel="dns-prefetch" href="//(.*?):', content[0])
ports = re.findall(':(.*?)">', content[0])
for ip, port in zip(ips, ports):
self.proixes[ip] = port
elif item[0] == 'daili89':
content = re.findall('<td>(.*?)</td>', response, re.S)
ips = [content[index].strip('\n').strip('\t') for index in range(0, len(content), 5)]
ports = [content[index].strip('\n').strip('\t') for index in range(1, len(content), 5)]
for ip, port in zip(ips, ports):
self.proixes[ip] = port
elif item[0] == 'zhandaye':
ips = re.findall('<a href="/ip/CheckHttp/(.*?)" title=', response)
for i in ips:
self.proixes[i.split(':')[0]] = i.split(':')[1]
self.index+=1
self.progress_bar('爬取IP进度',len(self.url_list))
async def check_proxy(self,proxy):
timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession(trust_env=True) as conn:
async with conn.get(self.target_url,timeout=timeout,proxy=f'http://{proxy[0]}:{proxy[1]}',ssl=False) as res:
self.index += 1
response = await res.text()
if str(res.status)[0] == '2' and response:
self.good_proixes[proxy[0]]=proxy[1]
self.progress_bar('成功率', len(self.proixes))
def progress_bar(self,msg,length):
print('\r',f"{msg}: {int((self.index/length)*100)}%: ", "▋" * int((self.index / length)*100), end="")
async def main(self):
self.kuaidaili()
self.xiaohuan()
self.ip66()
self.dieniao()
self.kaixin()
self.yundaili()
self.freedaili()
self.daili89()
self.zhandaye()
tasks = [asyncio.ensure_future(self.crawl_ip(item)) for item in self.url_list]
await asyncio.gather(*tasks,return_exceptions=True)
self.index=0
tasks=[asyncio.ensure_future(self.check_proxy(proxy)) for proxy in self.proixes.items()]
await asyncio.gather(*tasks,return_exceptions=True)
def entrance(self):
loop=asyncio.get_event_loop()
loop.run_until_complete(self.main())
if __name__ == '__main__':
a=Proxy()
a.entrance()
print(a.good_proixes)