爬虫收集上百条代理ip 代码分享！

# coding=utf-8
import urllib2
import time
from lxml import etree

#是通过西刺网站来获取代理信息
url = "http://www.xicidaili.com/"
header = {
    "User-Agent": 'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50',
}

request = urllib2.Request(url, headers=header)
data = urllib2.urlopen(request).read()
print(data)
content = etree.HTML(data)　　#xpath 来解析页面！　转成可以解析的页面！
url_list = content.xpath('//tr/td[2]/text()')
port = content.xpath('//tr/td[3]/text()')
text = content.xpath('//td[6]/text()')


li = list()
for i in range(len(url_list)):
    li.append(url_list[i] + ":" + port[i])
print(li)

 

转载于:https://www.cnblogs.com/shi-qi/articles/8906337.html