tieba_class

import urllib2, re

class Tools(object):
    remove_emement = re.compile(r'<.*?>', re.S)
    remove_space = re.compile(r' ', re.S)
    remove_n = re.compile(r'\n', re.S)
    replace_br = re.compile(r'<br>|<br/>')

    def update_data(self, res_tuple):
        name = re.sub(self.remove_n, '', res_tuple[0])
        name = re.sub(self.remove_emement, '', name)
        name = re.sub(self.remove_space, '', name)

        content = re.sub(self.remove_n, '', res_tuple[1])
        content = re.sub(self.remove_emement, '', content)
        content = re.sub(self.remove_space, '', content)
        content = re.sub(self.replace_br, '\n', content)

        return (name, content)

定义百度贴吧爬虫类

class BDTB(object):
    def __init__(self, numbers):
        # 基础路径及请求头
        self.base_url = "http://tieba.baidu.com/p/"+numbers
        self.headers = {
            'User-Agent':"Mozilla/5.0(Macintosh;IntelMacOSX10.6;rv:2.0.1)Gecko/20100101Firefox/4.0.1"
        }

        # 初始化工具类对象
        self.tool = Tools()

    # 定义获取网页源代码的函数，设置一个形参，用于接收页码
    def get_page_code(self, page_number):
        abs_url = self.base_url + '?pn=' + str(page_number)
        request = urllib2.Request(abs_url, headers=self.headers)
        try:
            response = urllib2.urlopen(request)
        except Exception, e:
            print '获取第%s页数据失败，原因是:%s'%(page_number, e)
            return None
        else:
            return response.read()

    # 解析源代码
    def get_data(self, html):
        title_pattern = re.compile(r'<h1 class="core_title_txt.*?>(.*?)</h1>', re.S)
        title = re.search(title_pattern, html).group(1)
        numbers_pattern = re.compile(r'<li class="l_reply_num".*?<span class="red">(.*?)</span>', re.S)
        total_pages = re.search(numbers_pattern, html).group(1)
        return title, total_pages

    # 定义获取所有回复内容的函数
    def get_comments_data(self, html):
        comment_pattern = re.compile(r'<a.*?class="p_author_name.*?>(.*?)</a>.*?<div id="post_content.*?>(.*?)</div>', re.S)
        results_list = re.findall(comment_pattern, html)
        data_list = []
        for res_tuple in results_list:
            # 将元组传递到工具类中进行过滤
            news_tuple = self.tool.update_data(res_tuple)
            data_list.append(news_tuple)
        return data_list

    # 定义用于写入数据的函数
    def write_data(self, news_data, f):
        for name, content in news_data:
            # 形参news_data接收过滤后的数据，将数据进行保存
            f.write('用户昵称:{}'.format(name))
            f.write('\n')
            f.write('回复:{}'.format(content))
            f.write('\n')


    # 定义开始爬虫程序的函数
    def start_spider(self):
        # 先爬取第一页的内容，获取总页数
        html = self.get_page_code(1)
        if html == None:
            print '链接贴吧失败'
            return
        else:
            title, total_page = self.get_data(html)
            # 开始下载该贴的内容
            # print '要爬取的总页数：%s页' % total_page
            # 打开本地文件
            # 将title标题中的空格删除
            space = re.compile(' ')
            title = re.sub(space, '', title)
            title = re.sub(re.compile('/'), '', title)
            title = re.sub(re.compile('\n'), '', title)
            file_test = open('{}.txt'.format(title).decode('utf-8'), 'w')
            for x in xrange(1, 2):
                # print '正在爬取第%s页数据...'%x
                html = self.get_page_code(x)
                if html == None:
                    # 当前页面数据获取失败
                    continue
                results = self.get_comments_data(html)
                self.write_data(results, file_test)
            file_test.close()

if __name__ == '__main__':
    bdtb = BDTB('5147943292')
    bdtb.start_spider()