python爬虫实战
需求:输入吧名,输入起止页码,创建一个文件夹,存储吧名_page.html文件
import urllib.parse
import urllib.request
import os
# 需求:输入吧名,输入起止页码,创建一个文件夹,存储吧名_page.html文件
# http://tieba.baidu.com/f?ie=utf-8&kw=python&pn=100
url='http://tieba.baidu.com/f?ie=utf-8&'
name=input('请输入你要查询的吧名名称:')
start_page=int(input('请输入你要查询起始页码:'))
end_page=int(input('请输入你要查询结束页码:'))
# name='python'
# start_page=2
# end_page=3
# 创建文件件
if not os.path.exists(name):
os.mkdir(name)
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
}
for page in range(start_page,end_page+1):
data={
'kw':name,
'pn':(page-1)*50
}
query_string=urllib.parse.urlencode(data)
url_get = url + query_string
request=urllib.request.Request(url=url_get,headers=headers)
print('第%s页开始下载......' % page)
response=urllib.request.urlopen(request)
# 生成文件名
filename=name+'_'+ str(page) +'.html'
filepath=name+'/'+filename
with open(filepath,'wb') as fp:
fp.write(response.read())
print('第%s页已经完成' % page)