利用python爬取百度贴吧源码

import requests

class tieba(object):
    def __init__(self,tieba_name):
        self.tieba_name = tieba_name
        self.start_url = 'https://tieba.baidu.com/f?kw=' + tieba_name + '&ie=utf-8&pn={}'
        self.headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) '
                                     'AppleWebKit/537.36 (KHTML, like Gecko) '
                                     'Chrome/70.0.3538.77 '
                                     'Safari/537.36'}

    def get_url_list(self):
        #构造url列表
        url_list = []
        for i in range(10):
            url_list.append(self.start_url.format(i*50))
        print(url_list)
        return url_list


    def parse_url(self,url):
        response = requests.get(url,headers = self.headers)
        return response.content.decode()


    def save_html(self,html,num):
        with open('英雄联盟贴吧源码/'+self.tieba_name+'-'+"第"+str(num)+"页.html",'w',encoding='utf-8') as f:
            f.write(html)


    def run(self):
        # 构造url列表
        url_list = self.get_url_list()
        for url in url_list:
            print(url)
            #发送请求，得到响应
            html = self.parse_url(url)
            page_num = url_list.index(url)+1
            #保存
            self.save_html(html,page_num)


if __name__ == '__main__':
    tb = tieba("英雄联盟")
    tb.run()

效果如下：

最下面main方法中可以传任意参数，各种贴吧名字，自己保存的时候是可以看到源码的，但是如果点击在浏览器中是打不开整个网页的，百度做了防爬处理，把大部分源码都写在了注释里面<!-- >,在通过json做处理展现整个网页