python实例017--利用python爬取网站上的代理ip

利用python爬取网站上的代理ip 爬取结果保存在一个列表中 存储在 agent.pkl 中,可以利用pickle模块加载

# __*__coding:UTF-8 _*_

import urllib.request
import re
import pickle
import os

#url="http://ip.yqie.com/ipproxy.htm"
#url="https://www.89ip.cn/index_25.html"

#注意 返回对象未进行解码
def openUrl(url):
    req=urllib.request.Request(url)
    #设置访问头
    req.add_header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36")

    #获得网页对象
    response=urllib.request.urlopen(req)
    html=response.read()
    return html

#解析读取网页中所有的代理地址
def getAgents(html):
    html=html.decode("utf-8")
    #print(html)
    #匹配 ip地址 正则表达式
    pattern = re.compile(r'(<td>)\s*((25[0-5]|2[0-4]\d|[0-1]\d\d|\d\d|\d)\.){3}(25[0-5]|2[0-4]\d|[0-1]\d\d|\d\d|\d)\s*</td>')
    ipList=[]
    ip=pattern.finditer(html)
    for ipiter in ip:
        ipText=ipiter.group()
        ipGroup=re.search(r"((25[0-5]|2[0-4]\d|[0-1]\d\d|\d\d|\d)\.){3}(25[0-5]|2[0-4]\d|[0-1]\d\d|\d\d|\d)", ipText)
        ipList.append(ipGroup.group())

    #匹配 端口地址 正则表达式
    portList=[]
    pattern = re.compile(r'(<td>)\s*\d+\s*</td>')
    port = pattern.finditer(html) 
    for portiter in port:
        portText=portiter.group()
        portGroup=re.search(r"\d+", portText)
        portList.append(portGroup.group())

    if(len(ipList) is not len(portList)):
        print("注意: ip和端口参数不匹配!")
        return
    ipDict=dict(zip(ipList,portList))

    agentList=[]
    for key in ipDict:
        agentList.append(key+":"+ipDict.get(key))
    return agentList

def writeAgentList(agentList): 
    if os.path.exists("agent.pkl"):
        os.remove("agent.pkl")          #每次重新生成 要不多次 dump需要多次 load
    with open("agent.pkl.","wb") as f:
        pickle.dump(agentList, f)

def loadAgentList():
    agentlist=[]
    if not os.path.exists("agent.pkl"):
        return agentlist
    with open("agent.pkl","rb") as f:
        agentlist=pickle.load(f)
    return agentlist
    
    

if __name__ == "__main__":
    page=input("爬取前多少页的ip?")
    indexRe = re.search(r"\d+", page)
    if(not indexRe):
        print("输入页数有误!")
    indexRe=int(indexRe.group())
    indexCur=1
    ipAgentList=[]
    while indexCur<indexRe:
        url="https://www.89ip.cn/index_{}.html".format(indexCur)
        print(url)
        html=openUrl(url)
        ipAgentList=loadAgentList()
        ipAgentList.extend(getAgents(html))
        writeAgentList(ipAgentList)
        indexCur+=1
    attArray=loadAgentList()
    print(attArray)