python爬取快代理IP并测试IP的可用性

最新推荐文章于 2021-06-15 11:24:15 发布

程序员班长

最新推荐文章于 2021-06-15 11:24:15 发布

阅读量676

点赞数 1

分类专栏： Python 文章标签：爬虫

本文链接：https://blog.csdn.net/KK_2018/article/details/112882517

版权

Python 专栏收录该内容

85 篇文章 20 订阅

订阅专栏

用到的网站https://www.kuaidaili.com/，免费的IP很不稳定，随时会挂，有需求的还是购买付费IP比较稳

import requests
from urllib import parse
from bs4 import BeautifulSoup

headers={
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
}

session=requests.session()
session.headers=headers

# 获取IP地址页面
def getIP(url):
    html=requests.get(url)
    # print(html.status_code)
    # print(html.text)
    if html.status_code==200:
        html.encoding=html.apparent_encoding
        soup=BeautifulSoup(html.text,'lxml')
        trs=soup.select('tbody tr')
        for tr in trs:
            ip=tr.select('td')[0].text
            port=tr.select('td')[1].text
            type=tr.select('td')[3].text
            TestIP(ip,port,type)

# 测试IP代理的可用性，并将可用IP写入文件
def TestIP(ip,port,type):
    url="https://www.baidu.com"
    proxies={
        'http':'{}://{}:{}'.format(type,ip,port),
        'https':'{}://{}:{}'.format(type,ip,port)
    }
    # print(proxies)
    try:
        re=session.get(url,proxies=proxies,timeout=2,verify=False)
        print(re.status_code)
        print("可用IP为{}://{}:{}".format(type,ip,port))
        f.write("{}://{}:{}\n".format(type,ip,port))    # 写入文件
    except:
        print("不可用IP为{}://{}:{}".format(type,ip,port))

if __name__ == '__main__':
    t=input("请输入要爬取的页数，每页15个：")
    url1="https://www.kuaidaili.com/free/intr/"
    with open('IP代理.txt','w') as f:
        for i in range(int(t)):
            url=parse.urljoin(url1,str(i+1))
            print(url)
            getIP(url)