Python零基础之用三种方式爬取百度贴吧(urllib,request,parse)

一: 入门写法

# !/usr/bin/python
# Filename: 实战 爬取百度贴吧.py
# Data    : 2020/07/14
# Author  : --king--
# ctrl+alt+L自动加空格格式化

import urllib.request, urllib.parse
import random


# 随机获取一个user-agent
# 百度上很多U-A大全
header_list = [{
                   'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36'},
               {
                   'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'},
               {
                   'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16'}
               ]

# 目标: 爬取百度贴吧的html代码
# 1. 输入需要爬取的贴吧的主题
# https://tieba.baidu.com/f?kw=%E4%B8%AD%E5%9B%BD&ie=utf-8&pn=0
# https://tieba.baidu.com/f?kw=%E4%B8%AD%E5%9B%BD&ie=utf-8&pn=50
# 规律是pn+50,第n页为

# 2. 输入需要爬取的起始页和终止页

# 3. 把每一页的内容保存到本地

# 从header_list里边随机选择一个作为headers
headers = random.choice(header_list)

name = input('请输入贴吧名:')

start = int(input('请输入起始页:'))

end = int(input('请输入结束页:'))

# 对想要搜索的贴吧名进行编码
kw = {'kw': name}

kw = urllib.parse.urlencode(kw)

# 开始拼接url 发起请求 获取响应
for i in range(start, end + 1):
    print(i)

    # 开始拼接url
    pn = (i - 1) * 50

    # kw后不用写等号了,字典翻译后会自动加等号
    baseurl = 'https://tieba.baidu.com/f?'

    # pn要改编为str类型才能正常拼接
    url = baseurl + kw + '&pn=' + str(pn)

    # 发起请求
    req = urllib.request.Request(url, headers=headers)

    # 获取响应
    res = urllib.request.urlopen(req)

    # 解码为utf-8
    html = res.read().decode('utf-8')

    # 写入文件
    filename = '第' + str(i) + '页.html'

    with open(filename, 'w', encoding='utf-8') as f:
        print('正在爬取第%d页' % i)
        f.write(html)

二: 用函数的方法

# !/usr/bin/python
# Filename: 实战 爬取百度贴吧2.py
# Data    : 2020/07/14
# Author  : --king--
# ctrl+alt+L自动加空格格式化

import random
import urllib.request, urllib.parse


# 读取页面
def readPage(url):
    header_list = [{
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36'},
        {
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'},
        {
            'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16'}
    ]

    headers = random.choice(header_list)

    req = urllib.request.Request(url, headers=headers)

    res = urllib.request.urlopen(req)

    html = res.read().decode('utf-8')
    return html


# 写入文件
def writePage(filename, html):
    with open(filename, 'w', encoding='utf-8') as f:
        f.write(html)
        print('写入成功')


# 主函数
def main():
    name = input('请输入贴吧名:')

    start = int(input('请输入起始页:'))

    end = int(input('请输入结束页:'))

    # 对贴吧name多编码
    kw = {'kw': name}

    kw = urllib.parse.urlencode(kw)

    for i in range(start, end + 1):
        # 开始拼接url
        pn = (i - 1) * 50

        # kw后不用写等号了,字典翻译后会自动加等号
        baseurl = 'https://tieba.baidu.com/f?'

        # pn要改编为str类型才能正常拼接
        url = baseurl + kw + '&pn=' + str(pn)

        html = readPage(url)

        # 写入文件
        filename = '第' + str(i) + '页.html'

        writePage(filename, html)


if __name__ == '__main__':
    main()

三: 类的方法

# !/usr/bin/python
# Filename: 实战 爬取百度贴吧3.py
# Data    : 2020/07/14
# Author  : --king--
# ctrl+alt+L自动加空格格式化


import urllib.request
import urllib.parse
import random

class BaiduSpider:

    def __init__(self):
        # 定义headers列表
        self.header_list = [{
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36'},
        {
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'},
        {
            'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16'}
    ]
        self.headers=random.choice(self.header_list)
        self.baseurl = 'https://tieba.baidu.com/f?'

    def readPage(self,url):

        req = urllib.request.Request(url, headers=self.headers)

        res = urllib.request.urlopen(req)

        html = res.read().decode('utf-8')

        return html

    def writePage(self,filename,html):

        with open(filename, 'w', encoding='utf-8') as f:
            f.write(html)
            print('写入成功')

    def main(self):

        name = input('请输入贴吧名:')

        start = int(input('请输入起始页:'))

        end = int(input('请输入结束页:'))

        # 对贴吧name多编码
        kw = {'kw': name}

        kw = urllib.parse.urlencode(kw)

        for i in range(start, end + 1):

            # 开始拼接url
            pn = (i - 1) * 50

            url = self.baseurl + kw + '&pn=' + str(pn)

            html = self.readPage(url)

            filename = '第' + str(i) + '页.html'

            self.writePage(filename,html)

if __name__ == '__main__':

    # 创建类的实例spider
    spider = BaiduSpider()

    # 调用spider实例对象中的main函数
    spider.main()

需要注意的几点问题

• 字典翻译过来后会自动添加=等号
• 类的中函数的元素创建都用self.的形式,调用的时候也是同样
• 注意返回值return的使用和函数对象的传参