python3.4-小爬虫2

#coding:UTF-8
__author__ = 'holoto'
from urllib import request
import re
from bs4 import BeautifulSoup
def user_agent(user):
    a={}
    a["chrome17"]="User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"
    a["Maxthon"]="User-Agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)"
    a['360']="User-Agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)"
    a["firefox"]='Mozilla/5.0 (X11; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0'
    a["G3_chrome"]='Mozilla/5.0 (Linux; Android 4.4.4; LG G3 Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.89 Mobile Safari/537.36'
    a['iphone']='Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/7.0 Mobile/12A365 Safari/9537.53'
    return a[user]
def add_user_agent(url,useragent):
    url.add_header('User-Agent',useragent)
def proxy_get(ll):#获取免费的代理ip
    #nn 本地高匿
    #nt 本地
    #wn 国外高匿
    #wt 国外
    proxy_url='http://www.xicidaili.com/'+ll
    proxy_req=request.Request(proxy_url)
    proxy_useragent=user_agent('G3_chrome')
    add_user_agent(proxy_req,proxy_useragent)
    f=request.urlopen(proxy_req)
    ff=f.read().decode()
    soup=BeautifulSoup(ff,"html5lib")
    proxy_max=10
    proxy_ip=soup.find_all(text=re.compile("\d*\.\d*\.\d*\.\d*"),limit=proxy_max)#代理ip
    proxy_port=soup.find_all(text=re.compile("^\d{2,4}$"),limit=proxy_max)#代理端口
    proxy_type=soup.find_all(text=re.compile("^HTTPS{0,1}$"),limit=proxy_max)#网络参数
    print([proxy_ip,proxy_port,proxy_type])
    return [proxy_ip,proxy_port,proxy_type]
def ss_get():#获取免费的ss帐号
    ss_url='http://www.ishadowsocks.com/'
    ss_req=request.Request(ss_url)
    ss_useragent=user_agent('firefox')
    add_user_agent(ss_req,ss_useragent)
    ss_g=request.urlopen(ss_req)
    ss_read=ss_g.read().decode()
    soup=BeautifulSoup(ss_read,"html5lib")
    data=soup.find_all(text=re.compile('\w*\:\S*$')).__str__()
    data1=re.findall('\:\S*',data)
    del data1[4]
    del data1[8]
    del data1[12]
    account=[data1[0][1:-2],data1[4][1:-2],data1[8][1:-2]]#帐号
    account_pwd=[data1[2][1:-2],data1[6][1:-2],data1[10][1:-2]]#密码
    account_port=[data1[1][1:-2],data1[5][1:-2],data1[9][1:-2]]#端口
    account_type=[data1[3][1:-2],data1[7][1:-2],data1[11][1:-2]]#网络参数
    return [account,account_port,account_pwd,account_type]#返回
#ss_get()
proxy_get('wt')