使用python解析网页内容



估计很多人都用过了 Firefox 自带的Web开发者工具来诊断或调试网页，尤其是网站建设人员。该工具非常强大，当我们想研究一张网页的访问详情时，例如想知道网页包含有哪些请求，各请求的访问是否正常，访问时间是怎样的等等，那么我们就可以借助于该工具。Firefox自带的Web开发者工具使用非常简单，点击一下按钮就可以窥视到网页内部各元素的瀑布流式请求，可以看到网页里哪些文件在什么时候被请求和耗时多少并显示到浏览器上。

点击菜单->工具->Web开发者->切换工具， 默认的Web开发者工具包含控制台、查看器、调试器、央视编辑器、分析器和网络分析器这6个部分，本文要说的是网络分析器这部分。点击Web开发者工具的“网络”标签，会看到下面出现一个空表单头，包含的内容有方法、文件、域名、类型、大小和时间线，在最下面还有几个标签可以切换：所有、HTML、CSS、JS、XHR、字体、图片、媒体、Flash。

Python Class to GetWeb

class GetWebResult():
    def __init__(self):
        jobs_id = 139366
        self.jobs_url = 'http://cbsp-wm-bl01/icase/api/rest/test/jobresults/?ids=%d' % jobs_id

    @staticmethod
    def get_soup(url):
        headers = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}
        req = urllib2.Request(url, headers=headers)
        f = urllib2.urlopen(req, timeout=10)
        html = f.read()
        f.close()
        return html

    def main(self):
        save_path = os.path.join(os.getcwd(), 'JobResult.xls')
        string_of_results = self.get_soup(self.jobs_url)
        tmp = string_of_results.replace('null', "\"null\"").replace('true', "\"true\"").replace('false', "\"false\"")
        results = eval(tmp)[0].get('results')
        list_of_dict_result = []
        for result in results:
            list_of_dict_result.append(self.convert_result(result))
        key_list = ['Name', 'Image', 'Subsystem', 'Functionality', 'Objective', 'Assumptions', 'Steps', 'Tickets', 'Metrics', 'Notes', 'Result']
        xls = Excel(save_path)
        xls.insert_sheet(sheet_name='results', list_dict=list_of_dict_result, list_key=key_list)

    @staticmethod
    def convert_result(dict):
        dict_result = {}
        case = dict.get('case')
        dict_result['Tickets'] = dict.get('tickets')
        dict_result['Metrics'] = dict.get('metrics')
        dict_result['Notes'] = dict.get('notes')
        dict_result['Result'] = dict.get('result')
        dict_result['Name'] = case.get('name')
        dict_result['Image'] = case.get('image').get('name')
        dict_result['Subsystem'] = case.get('taxonomy').get('subsystem').get('name')
        dict_result['Functionality'] = case.get('taxonomy').get('functional_area').get('name')
        dict_result['Objective'] = case.get('objective').decode('utf-8')
        dict_result['Assumptions'] = case.get('assumptions')
        dict_result['Steps'] = case.get('steps').decode('utf-8')
        return dict_result

Python Code to operate the Excel

class Excel():
    def __init__(self,path):
        self.book = xlwt.Workbook()
        self.path = path
        self.STYLE_LEFT_CENTER= self.leftcenterStyle()
        self.STYLE_CENTER = self.centerStyle()

    def insert_sheet(self, sheet_name, list_dict, list_key):
        new_sheet = self.book.add_sheet(sheetname=sheet_name, cell_overwrite_ok=True)
        for key in list_key:
            new_sheet.write(0, list_key.index(key), key, self.STYLE_CENTER)
        line = 0
        for row_value in list_dict:
            line += 1
            for key in list_key:
                new_sheet.write(line, list_key.index(key), row_value.get(key), self.STYLE_LEFT_CENTER)
            try:
                self.book.save(self.path)
            except Exception:
                for key in list_key:
                    new_sheet.write(line, list_key.index(key), '', self.STYLE_LEFT_CENTER)
                for key in list_key:
                    try:
                        new_sheet.write(line, list_key.index(key), row_value.get(key), self.STYLE_LEFT_CENTER)
                        self.book.save(self.path)
                    except Exception,e:
                        print e
                        print 'LogException===============>>>>>>>>  NAME = ' + key
                        print 'LogException===============>>>>>>>>  VALUE= '
                        print row_value.get(key)
                        new_sheet.write(line, list_key.index(key), 'ERROR,PLEASE CHECK MANUALLY!!', self.STYLE_LEFT_CENTER)
                        self.book.save(self.path)


    def leftcenterStyle(self):
        style = xlwt.XFStyle()
        style.alignment=self.LeftCenter_Alignment()
        style.borders = self.Thin_BORDER()
        return style
    def centerStyle(self):
        style = xlwt.XFStyle()
        style.alignment=self.Center_Alignment()
        style.borders = self.Thin_BORDER()
        return style
    def Center_Alignment(self):
        alignment = xlwt.Alignment()
        alignment.vert =alignment.VERT_CENTER
        alignment.horz =alignment.HORZ_CENTER
        return alignment

    def LeftCenter_Alignment(self):
        alignment = xlwt.Alignment()
        alignment.vert =alignment.VERT_CENTER
        alignment.horz =alignment.HORZ_LEFT
        return alignment
    def Thin_BORDER(self):
        borders =xlwt.Borders()
        borders.left = xlwt.Borders.THIN
        borders.right = xlwt.Borders.THIN
        borders.top = xlwt.Borders.THIN
        borders.bottom = xlwt.Borders.THIN
        return borders