利用代理IP爬取网页的小心得

---

利用爬虫爬取网页时，有时候会因为请求次数过多而被封IP，这时候就需要多个代理IP不断更换以保持爬虫继续工作。昨天遇到这个问题于是去查代理IP使用的问题，找到了这个：https://github.com/Germey/ProxyPool
网上大佬推荐的，但是我完全看不懂而且我电脑也不知道为什么运行报错，于是自己写了一个无比无比简陋的。。萌新可以有需要看一看。

import requests
from requests import RequestException
from bs4 import BeautifulSoup
import random
url = “http://www.xicidaili.com/”
def get_html(url):
header = {
‘User-Agent’: ‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36’
}
try:
response = requests.get(url, headers=header)
if response.status_code == 200:
return response.text
return None
except RequestException:
return None

def parse_html(html):
soup = BeautifulSoup(html, ‘lxml’)
str = ”
count = 0
for td in soup.select(‘td’):
str = str + td.get_text() + ’ ’
count += 1
if count == 8:
with open(‘Proxy_IP.txt’, ‘w’) as f:
f.write(str + ‘\n’)
count = 0
str = ”

def parse_proxy():
type = ‘HTTP’
with open(‘Proxy_IP.txt’, ‘r’) as f:
proxies = f.readlines()
for proxy in proxies:
proxy = proxy.split(’ ‘)
if str(proxy[5]) == type:
ip = proxy[1] + ‘:’ + proxy[2]
with open(‘Proxy_Pool.txt’, ‘a’) as fw:
fw.write(ip + ‘\n’)

def get_proxy():
with open(‘Proxy_Pool.txt’, ‘r’) as fw:
proxies = fw.readlines()
total_line_num = len(proxies)
line_num = random.randint(0, total_line_num)
return proxies[line_num]

def main():
html = get_html(url)
parse_html(html)
parse_proxy()
print(get_proxy())

if name == ‘main‘:
main()

然后这是在其他地方调用的函数：
def get_proxy():
path = r”G:\PyCharm\Project\Proxy_Pool.txt”
with open(path, ‘r’) as fw:
proxies = fw.readlines()
total_line_num = len(proxies)
line_num = random.randint(0, total_line_num)
return str(proxies[line_num]).strip(‘\n’)

因为水品十分有限所以目前只能实现这样的功能了，所幸还是能用的。。不过每次用之前最好将上面的重新跑一次更新一下Proxy_Pool.txt的内容。。