spider for bdtb

模仿中学习
python爬虫系列

代码：

# coding: utf-8
import re
import urllib2


class ContentTool:
    repl_t = re.compile('<td>')
    repl_n = re.compile('tr>|<div>|</div>|</p>|<p.*?>|</?br>')
    remo = re.compile('<.*?img.*?>| {7}|</?a.*?>|<.*?>')
    def rr(self, x):
        x = re.sub(self.repl_n, '\n', x)
        x = re.sub(self.repl_t, '\t', x)
        x = re.sub(self.remo, "", x)
        return x.strip()


class BaiduTieba:
    def __init__(self, baseURL, see_LZ, floorTag):
        self.baseUrl = baseURL
        self.see_lz = '?see_lz=' + str(see_LZ)
        self.tool = ContentTool()
        self.file = None
        self.floor = 1
        self.floorTag = floorTag

    def getPagecode(self, pagenum):
        try:
            url = self.baseUrl + self.see_lz + '&pn=' + str(pagenum)
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            return response.read().decode('utf-8')
        except urllib2.URLError, e:
            if hasattr(e, 'reason'):
                print u"百度贴吧链接失败，原因:", e.reason
                return None

    def getTitle(self, pagecode):
        expression = 'core_title_bg j_core_title_bg.*?title="(.*?)"'
        pattern = re.compile(expression, re.S)
        title = re.search(pattern, pagecode)
        if not title:
            self.file = open("百度贴吧" + ".txt", "w+")
        else:
            self.file = open(title.group(1).strip() + ".txt", "w+")

    def getPagenum(self, pagecode):
        expression = '<div class="pb_footer.*?<li class="l_reply_num".*?red">(.*?)</span>'
        pattern = re.compile(expression, re.S)
        pagenum = re.search(pattern, pagecode)
        if pagenum:
            return pagenum.group(1).strip()
        else:
            return None

    def getContent(self, pagecode):
        expression = '<div class="d_post_content_main.*?<cc>.*?class="d_post_content j_d_post_content ">(.*?)</div>'
        pattern = re.compile(expression, re.S)
        items = re.findall(pattern, pagecode)
        contents = []
        for item in items:
            content = "\n" + self.tool.rr(item) + "\n"
            contents.append(content.encode("utf-8"))
        return contents

    def writeData(self, contents):
        for item in contents:
            if self.floorTag == 1:
                floorline = "\n" + str(self.floor) + "___________________________________________________________\n"
                self.file.write(floorline)
            self.file.write(item)
            self.floor += 1

    def startOperation(self):
        pagecode = self.getPagecode(1)
        pagenum = self.getPagenum(pagecode)
        self.getTitle(pagecode)
        if not pagenum:
            print "URL已失效，请重试"
            return None
        try:
            print "该帖子共有" + str(pagenum) + "页"
            for i in range(1, int(pagenum) + 1):
                print "正在写入第" + str(i) + "页数据"
                page = self.getPagecode(i)
                contents = self.getContent(page)
                self.writeData(contents)
        except IOError, e:
            print "写入异常，原因" + e.message
        finally:
            print "写入任务完成"

baseURL = 'http://tieba.baidu.com/p/3138733512'
baidutieba = BaiduTieba(baseURL, 1, 1)
baidutieba.startOperation()

最后的内容可以改成人机交互的形式，这样可利用的价值就更大。
至于文中关于异常的处理，未实验，如果有哦问题，欢迎指出。