简介
- 抓取快代理-国内普通代理栏目;
- 默认抓取所有的数据,从第一页到最后一页;
- 抓取关键词: ip port;
- 时间间隔默认:2s,时间太短会导致抓取失败;
创建一个kuaiDaiLiGeneral.py
import os
import time
import requests
from bs4 import BeautifulSoup
class KuaiDaiLiGeneral(object):
def __init__(self):
self.session = requests.session()
self.proxies = None
self.timeout = 10
self.time_interval = 2
self.headers = {
"Accept": "text/html,application/xhtml+xml,"
"application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, sdch, br",
"Accept-Language": "zh-CN,zh;q=0.8",
"Connection": "Keep-Alive",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/55.0.2883.87 Safari/537.36",
}
def get_status(self, url):
"""
获取状态
:param url: 访问地址
:return: 返回response或False
"""
response = self.session.get(
url=url,
headers=self.headers,
proxies=self.proxies,
timeout=self.timeout,
)
if response.status_code == 200:
return response
else:
print("ERROR: 网络连接失败! status: %s url: %s" % (response.status_code, url))
print(response.text)
return False
def get_index(self, url):
"""
访问首页,建立连接
:param url:
:return:
"""
response = self.get_status(url)
if response:
print("首页,建立连接...")
return True
def get_last_page(self, url):
"""
获取最后一页page
:param url: 第一页的url
:return: 返回int(last_page)或None
"""
response = self.get_status(url)
if not response:
return None
html = response.text
soup = BeautifulSoup(html, "html5lib")
lis = soup.select("#listnav > ul > li")
if lis[-1].text == "页":
last_page = lis[-2].find("a").text
return int(last_page)
return None
def parse_html(self, url):
response = self.get_status(url)
if not response:
return None
html = response.text
soup = BeautifulSoup(html, 'html.parser')
items = soup.find(id="list").find("tbody").find_all("tr")
ip_port_list = []
for item in items:
ip = item.find(attrs={"data-title": "IP"}).text
port = item.find(attrs={"data-title": "PORT"}).text
ip_port = ip + ":" + port + "\n"
ip_port_list.append(ip_port)
return ip_port_list
@staticmethod
def write_to_text(path, content):
path = os.path.abspath(path)
with open(path, 'a+', encoding='utf-8') as f:
f.writelines(content)
def next_page(self, last_page):
for i in range(1, last_page + 1):
time.sleep(self.time_interval)
url = "https://www.kuaidaili.com/free/intr/{i}".format(i=i)
print(url)
ip_port_list = self.parse_html(url)
path = os.path.join(os.getcwd(), "IP.txt")
self.write_to_text(path, ip_port_list)
def main(self):
url = "https://www.kuaidaili.com"
if not self.get_index(url):
return None
time.sleep(self.time_interval)
url = "https://www.kuaidaili.com/free/intr/1/"
last_page = self.get_last_page(url)
if not last_page:
return None
self.next_page(last_page)
if __name__ == '__main__':
kuai_dai_li = KuaiDaiLiGeneral()
kuai_dai_li.main()