Python3爬虫urllib之善用模块-GET请求批量下载贴吧页面

接下来就直接进代码咯，开始吧！

# http://tieba.baidu.com/f?kw=python3&ie=utf-8  第一页 其实就是pn=0
# http://tieba.baidu.com/f?kw=python3&ie=utf-8&pn=50  第二页
# http://tieba.baidu.com/f?kw=python3&ie=utf-8&pn=100  第三页
#页码规律 页码=(n-1)*50
#导入相关的库
import urllib.request
import urllib.parse
import os

def handle_url(url,page,baming):
    # 拼接url
    pn = (page - 1) * 50
    data = {
        'kw':baming,
        'pn':pn
    }
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
    }
    data = urllib.parse.urlencode(data)
    url += data
    request = urllib.request.Request(url=url,headers=headers)
    return request
def download(request,page):
    response = urllib.request.urlopen(request)
    dirname_path = os.path.dirname(os.path.abspath(__file__)) #获取当前文件的目录名
    filename = '第'+str(page)+'页.html'
    filepath = os.path.join(dirname_path,filename)#拼接得到的目录名和文件名
    # print(filepath) #测试输出的路径
    with open(filepath,'w',encoding='utf-8') as fp:  #采用转码的方式写，就可以写入中文，否则就要写入二进制
        fp.write(response.read().decode('utf-8'))
def main():
    baming = input('请输入要爬取的贴吧名：')
    start_page = eval(input('请输入起始页码：'))
    end_page = eval(input('请输入结束页码：'))
    url = 'http://tieba.baidu.com/f?ie=utf-8&'
    for page in range(start_page,end_page + 1):
        request = handle_url(url,page,baming)
        print('开始下载第'+str(page)+'页.html')
        download(request,page)
        print('结束下载第' + str(page) + '页.html')

if __name__ == '__main__':
    main()