需求:输入吧名,输入起始页码,输入结束页码,然后再当前文件夹中创建一个以吧名为名字的文件夹,里面是每一页的html内容,文件名是吧名_page.html
import urllib.request
import urllib.parse
import os
url = 'http://tieba.baidu.com/f?ie=utf-8'
ba_name = input('请输入吧名:')
start_page = int(input('请输入要爬去的起始页码'))
end_page = int(input('请输入要爬去的结束页码:'))
#创建文件夹
os.mkdir(ba_name)
if not os.path.exists(ba_name):
os.mkdir(ba_name)
#一次爬去每一页数据
for page in range(start_page,end_page + 1):
# page就是当前页
#拼接url的过程
data ={
'kw': ba_name,
'pn':(page - 1)*50
}
data = urllib.parse.urlencode(data)
#生成指定的url
url_t = url + data
#print(url_t)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'
}
request = urllib.request.Request(url=url_t,headers=headers)
print('第%s页开始下载...' % page)
response = urllib.request.urlopen(request)
#生成文件名
filename = ba_name + '_' + str(page) + '.html'
#拼接文件路径
filepath = ba_name + '/' + filename
#写内容
with open(filepath,'wb') as fp:
fp.write(response.read())
print('第%s页结束下载...' % page)