#有些代理不能使用,使用前先写脚本筛选
import requests
from lxml import etree
import xlwt
import time
ip = xlwt.Workbook(encoding='utf-8')
sheet1 = ip.add_sheet('test1',cell_overwrite_ok=True)
sheet1.write(0,0,"type")
sheet1.write(0,1,"address")
sheet1.write(0,2,"port")
url = 'http://www.xicidaili.com/nn/'
user_agent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6)"
headers = {'User-Agent' : user_agent}
filepath = 'ip.xls'
num = 0
for i in range(1,10):
url = url + str(i)
headers = [
{'User-Agent' : user_agent},
{'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'},
{'Host': 'www.xicidaili.com'},
{'Accept-Encoding': 'gzip, deflate'}
]
r = requests.get(url , headers = headers)
time.sleep(5)
print(num)
data = etree.HTML(r.text)
cnt = data.xpath('//tr[@class="odd"]')
for i in cnt:
num += 1
address = i.xpath('.//td[2]/text()')[0]
port = i.xpath('.//td[3]/text()')[0]
Type = str(i.xpath('.//td[6]/text()')[0]).lower()
sheet1.write(num,0,Type)
sheet1.write(num,1,address)
sheet1.write(num,2,port)
cnt = data.xpath('//tr[@class=""]')
for i in cnt:
num +=1
address = i.xpath('.//td[2]/text()')[0]
port = i.xpath('.//td[3]/text()')[0]
Type = str(i.xpath('.//td[6]/text()')[0]).lower()
sheet1.write(num, 0, Type)
sheet1.write(num, 1, address)
sheet1.write(num, 2, port)
print(num)
ip.save(filepath)
爬虫获取西刺免费高匿代理
最新推荐文章于 2020-07-30 14:18:03 发布