获得代理ippython_Python学习笔记六(免费获取代理IP)

1 importrequests2 importurllib.request3 from bs4 importBeautifulSoup4 importrandom,time,re5 importrandom6 from urllib.request importFancyURLopener7

8 IPRegular = r"(([1-9]?\d|1\d{2}|2[0-4]\d|25[0-5]).){3}([1-9]?\d|1\d{2}|2[0-4]\d|25[0-5])"

9 headers = {'User-Agent', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36'}10 header1 = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)''AppleWebKit/537.36 (KHTML, like Gecko)''Ubuntu Chromium/44.0.2403.89'

11 'Chrome/44.0.2403.89''Safari/537.36'}12 #定义用于存放IP的列表

13 IPs =[]14 proxy_list =[]15

16

17 #代理一：

18 defgetIPList1():19 for i in range(1, 3):20 req = urllib.request.Request(url='http://www.xicidaili.com/nt/{0}'.format(i), headers=header1)21 r =urllib.request.urlopen(req)22 soup = BeautifulSoup(r,'html.parser',from_encoding='utf-8')23 table = soup.find('table', attrs={'id': 'ip_list'})24 tr = table.find_all('tr')[1:]25 #解析得到代理ip的地址，端口，和类型

26 for item intr:27 tds = item.find_all('td')28 temp_dict ={}29 kind = "{0}:{1}".format(tds[1].get_text().lower(), tds[2].get_text())30 proxy_list.append(kind)31 returnproxy_list32

33 #代理二：

34 defgetIPList2():35 #代理网站的地址的格式

36 #根据观察url，发现各省的代理IP页面由数字控制

37 #所以我们先用占位符{}代替这个数字的位置

38 url = 'http://www.66ip.cn/areaindex_16/{}.html'

39 for page in range(10):40 #先填占位符生成一个省的url

41 url =url.format(page)42 #get()方法访问，得到一个Response对象

43 rsp =requests.get(url)44 #Response对象的text属性得到源码

45 text =rsp.text46 #用BeautifulSoup()方法将源码生成能被解析的lxml格式文件

47 soup = BeautifulSoup(text, 'lxml')48 #用find()找放置IP的表

49 table = soup.find(name='table', attrs={'border': '2px'})50 #用find_all()找到所以的IP

51 ip_list = table.find_all(name='tr')52 #循环遍历每个IP

53 for addr inip_list:54 #观察源码发现第一个tr里的内容不是IP，所以跳过

55 if addr ==ip_list[0]:56 continue

57 #获取IP

58 ip = addr.find_all(name='td')[0].string59 #获取端口

60 port = addr.find_all(name='td')[1].string61 proxy = '%s:%s' %(ip, port)62 proxy_list.append(proxy)63 returnproxy_list64

65 #代理三：

66 defgetIPList3():67 request_list =[]68 headers ={69 'Host': 'www.iphai.com',70 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',71 'Accept': r'application/json, text/javascript, */*; q=0.01',72 'Referer': r'http://www.iphai.com',}73 request_item = "http://www.iphai.com/free/ng"

74 request_list.append(request_item)75 for req_id inrequest_list:76 req = urllib.request.Request(req_id, headers=headers)77 response =urllib.request.urlopen(req)78 html = response.read().decode('utf-8')79

80 ip_list = re.findall(r'\d+\.\d+\.\d+\.\d+', html)81 port_list = re.findall(r'(\s\d+\s)', html)82 for i inrange(len(ip_list)):83 #total_count += 1

84 ip =ip_list[i]85 port = re.sub(r'(^\s*)|(\s*$)/g','',port_list[i])86 proxy = '%s:%s' %(ip, port)87 proxy_list.append(proxy)88 returnproxy_list89

90

91

92 if __name__ == "__main__":93 #选择可以访问的代理

94 list1 = ['http://www.66ip.cn/','https://www.xicidaili.com','http://www.iphai.com/free/ng']95 while 1==1:96 for url inlist1:97 try:98 r = requests.get(url, timeout=5,headers=header1)99 if r.status_code == 200:100 if url ==list1[0]:101 print ("OK 网站访问正常", url)102 IPs =getIPList2()103 elif url == list1[1]:104 print ("OK 网站访问正常", url)105 IPs =getIPList1()106 elif url == list1[2]:107 print ("OK 网站访问正常", url)108 IPs =getIPList3()109 break

110 except:111 print ("Error 不能访问！", url)112 break

113 print("获取的代理IP为：",IPs)114 time.sleep(10)