python爬虫练习之用urllib模块来爬取百度贴吧指定内容指定页面的全网页内容，（版本2）

目标：

用urllib模块实现
根据搜索来获取指定贴吧内容网页，然后以’.html’格式直接保存网页，

可参考：

https://blog.csdn.net/xdc1812547560/article/details/107989273
[https://blog.csdn.net/xdc1812547560/article/details/108013490]
(https://blog.csdn.net/xdc1812547560/article/details/108013490)

第一步

引入模块

import urllib.request
import urllib.parse

读取页面

# 读取页面
def readPage(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36'
    }

    # 发起请求
    req = urllib.request.Request(url, headers=headers)

    # print(url)
    res = urllib.request.urlopen(req)

    html = res.read().decode('utf-8')

    return html

第二步

# 写入文件
def writePage(filename,html):

    with open(filename,'w',encoding='utf-8') as f:

        f.write(html)

第三步

主函数体

# 主函数
def main():

    name = input('请输入贴吧名:')

    start = int(input('请输入起始页:'))

    end = int(input('请输入结束页:'))

    # 对贴吧name进行编码
    kw = {'kw': name}

    kw = urllib.parse.urlencode(kw)

    for i in range(start,end+1):

        pn = (i - 1) * 50
        baseurl = 'https://tieba.baidu.com/f?'

        url = baseurl + kw + '&pn=' + str(pn)

        html = readPage(url)

        filename = '第' + str(i) + '页.html'

        writePage(filename,html)

运行

if __name__ == '__main__':

    main()

完整代码

import urllib.request
import urllib.parse

# 读取页面
def readPage(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36'
    }

    # 发起请求
    req = urllib.request.Request(url, headers=headers)

    # print(url)
    res = urllib.request.urlopen(req)

    html = res.read().decode('utf-8')

    return html

# 写入文件
def writePage(filename,html):

    with open(filename,'w',encoding='utf-8') as f:

        f.write(html)



# 主函数
def main():

    name = input('请输入贴吧名:')

    start = int(input('请输入起始页:'))

    end = int(input('请输入结束页:'))

    # 对贴吧name进行编码
    kw = {'kw': name}

    kw = urllib.parse.urlencode(kw)

    for i in range(start,end+1):

        pn = (i - 1) * 50
        baseurl = 'https://tieba.baidu.com/f?'

        url = baseurl + kw + '&pn=' + str(pn)

        html = readPage(url)

        filename = '第' + str(i) + '页.html'

        writePage(filename,html)


if __name__ == '__main__':

    main()

运行结果