对获取简单网页数据方法进行封装

对获取简单网页数据方法进行封装

# coding: utf-8
import urllib2
import urllib
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
import re

class BasicsQequest():

    def __init__(self):
        self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
        self.headers = {'User-Agent': self.user_agent}

    def basics_quest(self, url):
        try:
            # 构建请求的request
            request = urllib2.Request(url, headers=self.headers)
            # 利用urlopen获取页面代码
            response = urllib2.urlopen(request)
            # 将页面转化为UTF-8编码
            pageCode = response.read().decode('utf-8')
            # print pageCode
            return pageCode
        except urllib2.URLError, e:
            if hasattr(e, "reason"):
                print u"连接失败,错误原因", e.reason
                return None

    def basics_re(self, url, str_re):
        pageCode = self.basics_quest(url)
        if not pageCode:
            print "页面加载失败...."
            return None
        pattern = re.compile(str_re, re.S)
        items = pattern.findall(pageCode)
        # for item in items:
        #     print item
        return list(items)