python批量爬取网页数据_Python3-网页爬取-批量爬取贴吧页面数据

# 批量爬取贴吧页面数据# 网页抓取汉字转码、多个参数拼接# 第1页： https://tieba.baidu.com/f?kw=%E6%97%85%E8%A1%8C%E9%9D%92%E8%9B%99&ie=utf-8&pn=0# 第2页：https://tieba.baidu.com/f?kw=%E6%97%85%E8%A1%8C%E9%9D%92%E8%9B%99&ie=utf-8&pn=50# 第3页 https://tieba.baidu.com/f?kw=%E6%97%85%E8%A1%8C%E9%9D%92%E8%9B%99&ie=utf-8&pn=100# 第4页 pn=150# 及格水平---单页爬取# base_url = "https://tieba.baidu.com/f?kw=%E6%97%85%E8%A1%8C%E9%9D%92%E8%9B%99&ie=utf-8&pn="# for page in range(10):# new_url = base_url + str(page*50)# print(new_url)# 进阶水平--单页爬取# 从键盘去输入贴吧名称和页数，然后爬取指定页面的内容base_url = 'https://tieba.baidu.com/f?'name = input("请输入贴吧名称：")

page = input("请输入贴吧页数：") # page输入的时候就是字符串fromurllib importrequest,parse

# qs={'kw':name,# 'pn':(int(page)-1)*50}## qs_data=parse.urlencode(qs)# url=base_url+qs_data# print(url)## headers={# 'user-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'## }# req=request.Request(url,headers=headers)# response=request.urlopen(req)# html=response.read()# html=html.decode('utf-8')## with open(name+'第'+page+'页'+'.html','w',encoding='utf-8') as f:# f.write(html)# 进阶水平----批量爬取# 从键盘去输入贴吧名称和页数，然后爬取指定页面的内容fori inrange(int(page)):

qs = {'kw': name,'pn': i * 50}

qs_data = parse.urlencode(qs)

url = base_url + qs_data

print(url)

headers = {

'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'}

req = request.Request(url,headers=headers)

response = request.urlopen(req)

html = response.read()

html = html.decode('utf-8')

withopen(name + '第'+ str(i+1) + '页'+ '.html','w',encoding='utf-8') asf:

f.write(html)