python -V3.6
import random
import requests
from fake_useragent import UserAgent
from lxml import etree
user_agent = random.choice(UserAgent)
headers = {
'User-Agent':user_agent,
}
def gethtmlresponse():
# for i in range(1, 2):
url_fast = 'https://www.xicidaili.com/nn/1'
response = requests.get(url=url_fast, headers=headers)
return response.text
def gethtmlip(res):
etree_html = etree.HTML(res)
ip_name = etree_html.xpath('//table[@id="ip_list"]/tr[position()>1]/td[2]/text()')
ip_port = etree_html.xpath('//table[@id="ip_list"]/tr[position()>1]/td[3]/text()')
ip_http = etree_html.xpath('//table[@id="ip_list"]/tr[position()>1]/td[6]/text()')
print(ip_http) # ['HTTP', 'HTTP', 'HTTP', 'HTTPS', 'HTTP', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTP', 'HTTP', 'HTTP', 'HTTP', 'HTTP', 'HTTPS', 'HTTP', 'HTTP', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTP', 'HTTP', 'HTTP', 'HTTPS', 'HTTP', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTP', 'HTTP', 'HTTP', 'HTTP', 'HTTPS', 'HTTP', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTP', 'HTTP', 'HTTP', 'HTTP', 'HTTP', 'HTTP', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTP', 'HTTPS', 'HTTP', 'HTTP', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTP', 'HTTP', 'HTTP', 'HTTP', 'HTTPS', 'HTTP', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTP', 'HTTP', 'HTTP', 'HTTP', 'HTTPS', 'HTTP', 'HTTPS', 'HTTP', 'HTTPS', 'HTTP', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTP', 'HTTP', 'HTTPS', 'HTTP', 'HTTP', 'HTTP', 'HTTPS', 'HTTP', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS', 'HTTPS']
# exit()
for ip, port, mode in zip(ip_name, ip_port, ip_http):
item = '{}://{}:{}'.format(mode, ip, port)
ITEM = {
'item':item
}
# item = dict()
# item[mode] = '{}://{}:{}'.format(ip_http, ip_name, ip_port)
print(item)
print('aaa')
saveipcsv(ITEM['item'])
def saveipcsv(item):
with open('xici.csv','a',encoding='utf-8') as f:
f.write(item)
f.write('\n')
if __name__ == '__main__':
res = gethtmlresponse()
gethtmlip(res)