python爬虫学习——爬取糗百页面段子

糗百利用urllib2库进行操作，需要headers验证。
url是读取的页面。
写的程序可能因为糗百的网页结构更新导致正则表达式匹配错误，从而出错或者抓取不到。
程序敬请参考，有意见和问题欢迎讨论

#!/user/bin/env python
# coding=utf-8
import urllib
import urllib2
import re
import sys
defauletencoding = 'utf-8'
class QSBK:
    def __init__(self):
        self.pageIndex = 1
        self.user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:51.0) Gecko/20100101 Firefox/51.0"
        self.headers = {"User-Agent":self.user_agent}
        self.stories = []
    def getPage(self, pageIndex):
        try:
            url = "http://www.qiushibaike.com/hot/page/" + str(pageIndex) + "?s=4955345"
            request = urllib2.Request(url, headers=self.headers)
            response = urllib2.urlopen(request)
            context = response.read().decode("utf-8")
            return context
        except urllib2.URLError, e:
            if hasattr(e, "reason"):
                print e.reason
                return None

    def getPageItems(self, pageIndex):
        context = self.getPage(pageIndex)
        if not context:
            print "加载出错"
            return None
        pattern = re.compile(
            '<div class="article block untagged mb15".*?<h2>(.*?)</h2>.*?<div class="article.*?">(.*?)</div>.*?<span>(.*?)</span>.*?</a>(.*?)<div class="stats',
            re.S)
        items = re.findall(pattern, context)
        pageStories = []
        for item in items:
            havaImg = re.search("img", item[3])
            if not havaImg:
                replaceBR = re.compile("<br/>")
                text = re.sub(replaceBR, "\n", item[2])
                store = {"name":item[0], "age":item[1], "content":text}
                pageStories.append(store)
        return pageStories

    def start(self):
        print "开始读取段子"
        pageStories = self.getPageItems(1)
        for page in pageStories:
            print "昵称    :" + page['name']
            print "年龄    :" + page['age']
            print "段子    :" + page['content']
            print "\n\n"
reload(sys)
sys.setdefaultencoding(defauletencoding)
spider = QSBK()
spider.start()