爬虫-IP代理池构建

ip代理池构建（可自动调用）
多次爬取，进行ip替换可有效防止访问失败，本代码可直接导入使用

import requests
from lxml import etree
import time
import random
#随机选取代理
def getip(ipstock1):
	key1=random.choice(list(ipstock1))
	proxies2="https://"+key1+":"+ipstock1[key1]
	proxies3={"https":proxies2}
	return proxies3
#代理验证
def USEFUL(ipstock):
	url="http://www.baidu.com/"
	for key in list(ipstock.keys()):
		try:
			proxies="https://"+key+":"+ipstock[key]
			proxies1={"https":proxies}
			res=requests.get(url,proxies1)
			time.sleep(1)
		except Exception as e:
			print(e)
			ipstock.pop(key)
			continue
	return ipstock	
#解析内容，获取ip
def parse_content(url):

	headers={
	"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36",
	}
	r=requests.get(url,headers=headers)
	tree=etree.HTML(r.text)
	IP=tree.xpath('//table/tbody/tr/td[1]/text()')
	PORT=tree.xpath('//table/tbody/tr/td[2]/text()')
	return IP,PORT
	
def main(x,y,X,Y):
	IP2=[]
	PORT2=[]
	DAILI={}
	start_page=random.randint(x,y)
	end_page=random.randint(X,Y)
	#start_page=int(input("请输入爬取的起始页数："))
	#end_page=int(input("请输入爬取的结束页数："))
	#获取你想爬取的网页连接
	url="https://www.kuaidaili.com/free/inha/"#https://www.kuaidaili.com/free/inha/1/
	for i in range(start_page,end_page+1):
		link=url+str(i)+"/"
		print(link)
		IP1,PORT1=parse_content(link)
		IP2.extend(IP1)
		PORT2.extend(PORT1)
		time.sleep(3)
	ipstock=dict(zip(IP2,PORT2))
	ipstock1=USEFUL(ipstock)
	terminal_proxies=getip(ipstock1)
	print(terminal_proxies)

if __name__=="__main__":
	main()

本人新手，写的比较简单，也有许多冗余的地方，请大家多多指教