【python】自动采集有效IP代理

自动采集有效IP代理

#需要的外部依赖包requests和lxml
#自动获取的代理ip数据保存为”IP代理池.txt“
#read_ip函数用于提取”IP代理池.txt“的数据返回类型为列表
from lxml import etree
import requests
import datetime
not_alive=0
is_alive=0
def rm_symbol(c1):
    new_c1=c1.replace('\n','').replace('\t','')
    return  new_c1
#------------代理ip网站数据爬取(ip+空格+port形式)------------
def get_89ip_data():
    for i in range(1, 16):
        url = 'http://www.89ip.cn/index_{}.html'.format(i)
        res = requests.get(url)
        res.encoding = 'utf-8'
        html=etree.HTML(res.text)
        ipdress=html.xpath('//table[@class="layui-table"]/tbody/tr/td[1]/text()')
        port=html.xpath('//table[@class="layui-table"]/tbody/tr/td[2]/text()')
        ipdress=list(map(rm_symbol,ipdress))
        port=list(map(rm_symbol,port))
        data=list(zip(ipdress,port))
        for i in range(len(data)):
            ip_is_alive(data[i][0]+' '+data[i][1])
#----------------------------------------

#ip存活检测
def ip_is_alive(ip_port):
    import telnetlib
    global is_alive
    global not_alive
    ip=ip_port.split(' ')[0]
    port=ip_port.split(' ')[-1]
    try:
        tn = telnetlib.Telnet(ip, port=port,timeout=2)
    except:
        print('[-] ip:{}:{}'.format(ip,port))
        not_alive+=1
    else:
        print('[+] ip:{}:{}'.format(ip,port))
        is_alive+=1
        with open('IP代理池.txt','a') as f:
            f.write(ip+':'+port+'\n')

#提取IP池的ip 以列表形式返回
def read_ip(dress):
    with open(dress,'r') as f:
        ip_port=f.read().split('\n')[1:]
        for i in ip_port:
            if i=='':
                ip_port.remove(i)
    return ip_port

#检测过程
def check_ip():
    print('------------IP检测开始------------')
    print('[+]代表ip有效\n[-]代表ip无效\n')
    with open('IP代理池.txt', 'w') as f:
        f.write('------------自动获取 IP代理池-----------\n')
    get_89ip_data()
    with open('IP代理池.txt', 'a') as f:
        f.write('------更新时间：{}-----\n'.format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
    print("---总共检测{}个IP 有效IP:{}个 无效IP:{}个---".format(is_alive+not_alive,is_alive,not_alive))


check_ip()