python 爬虫获取有效的IP代理
通常在爬虫时,网站对于经常访问的IP会有所限制,因此我们需要一个IP代理来帮助我们来爬取网页,那么这些IP代理去哪里找呢?
西刺这个网站提供了很多免费的IP代理,但是这里面并不是都是有效的IP代理,因此还需要筛选,
(感觉西刺的不好用啊)
# @Time :2019/11/9 13:17
# @Auther :Ming
# @Software: PyCharm
import requests
from lxml import etree
def get_proxy():
proxy = []
for i in range(1,5):
url = 'https://www.xicidaili.com/nn/'+ str(i)
headers ={
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36'
}
reponse = requests.get(url= url,headers = headers)
html = reponse.text
html = etree.HTML(html)
ip = html.xpath('//tr/td[2]/text()')
post = html.xpath('//tr/td[3]/text()')
for a in range(len(ip)):
proxy.append(ip[a]+':'+post[a])
return proxy
def check(proxy):
url = 'http://www.baidu.com'
for proxy in proxy:
proxies = {
'http':'http://'+proxy,
'https': 'http://'+proxy,
}
try:
response = requests.get(url=url,proxies=proxies)
if response.status_code == 200:
with open('2.txt','a',encoding='utf-8') as f:
f.write(proxy+'\n')
except requests.exceptions.ConnectionError as e:
print('error',e.args)
# from selenium import webdriver
a = get_proxy()
check(a)