一次性付费进群,长期免费索取教程,没有付费教程。
进微信群回复公众号:微信群;QQ群:460500587
教程列表 见微信公众号底部菜单 | 本文底部有推荐书籍微信公众号:计算机与网络安全
ID:Computer-network
使用urllib2可以比较方便地处理那些无须验证码的登录页面,不过使用Mechanize登录更加方便。当然是怎么方便怎么做。下面以抓取光猫F460的设置页面为例,使用Mechanize配合bs4抓取光猫F460的数据。 1、获取F460数据 启动Eclipse,新建PyDev项目MechanizeAndBs4。在新项目中创建一个PyDev Module文件getF460Info.py。 getF460Info.py的内容如下:1 #!/usr/bin/evn python
2 #-*- coding: utf-8 -*-
3 '''
4 Created on
5
6 @author: hstking hst_king@hotmail.com
7 '''
8
9 import mechanize
10 from bs4 import BeautifulSoup
11 from mylog import MyLog as mylog
12
13
14 class F460Info(object):
15 '''获取光猫f460的信息 '''
16 def __init__(self):
17 self.url = 'http://192.168.1.1/'
18 self.log = mylog()
19 self.username = 'admin'
20 self.password = '******' #这里输入光猫的密码
21 self.spider()
22
23
24 def spider(self):
25 responseContent = self.getResponseContent(self.url)
26 if not responseContent:
27 self.log.error('the response is null')
28 exit()
29 soup = BeautifulSoup(responseContent, 'lxml')
30 modemInfo = {}
31 modemInfo['CarrierName'] = soup.find('td',attrs={'id':'Frm_CarrierName'}).get_text().strip()
32 modemInfo['modelName'] = soup.find('td',attrs={'id':'Frm_ModelName'}).get_text().strip()
33 modemInfo['SerialNumber'] = soup.find('td',attrs={'id':'Frm_SerialNumber'}).get_text().strip()
34 modemInfo['HardwareVer'] = soup.find('td',attrs={'id':'Frm_HardwareVer'}).get_text().strip()
35 modemInfo['SoftwareVer'] = soup.find('td',attrs={'id':'Frm_SoftwareVer'}).get_text().strip()
36 modemInfo['BootVer'] = soup.find('td',attrs={'id':'Frm_BootVer'}).get_text().strip()
37 modemInfo['VerDate'] = soup.find('td',attrs={'id':'Frm_VerDate'}).get_text().strip()
38
39 self.pipeline(modemInfo)
40
41
42 def getResponseContent(self, url):
43 self.log.info(u'begin create mechanize browser')
44 br = mechanize.Browser()
45 br.set_handle_equiv(True)
46 br.set_handle_gzip(False)
47 br.set_handle_redirect(True)
48 br.set_handle_referer(True)
49 br.set_handle_robots(False)
50 br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(),max_time=1)
51 br.addheaders = [('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64)AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36')]
52
53 self.log.info(u'open url on mechanize browser')
54 try:
55 br.open(url)
56 except:
57 self.log.error(u'open %s failed' %url)
58 return ''
59 br.select_form(nr=0)
60 br.form['Username'] = self.username
61 br.form['Password'] = self.password
62 br.submit()
63
64 newUrl = url + 'template.gch'
65 try:
66 br.open(newUrl)
67 except:
68 self.log.error(u'open %s failed' %newUrl)
69 return ''
70 else:
71 return br.response().read()
72
73
74 def pipeline(self, info):
75 fileName = u'f460ModemInfo.txt'.encode('gbk')
76 with open(fileName, 'w') as fp:
77 for key in info.keys():
78 print('%s \t %s \n' %(key.encode('utf8'),info.get(key).encode('utf8')))
79 fp.write('%s \t %s \n' %(key.encode('utf8'),info.get(key).encode('utf8')))
80
81
82 if __name__ == '__main__':
83 fi = F460Info()
然后将mylog.py复制到MechanizeAndBs4项目下,单击Eclipse图标栏的运行图标,执行结果如图1所示。微信公众号:计算机与网络安全
ID:Computer-network
【推荐书籍】