python3爬取贴吧的页面

爬取贴吧的页面

-- coding: utf-8 --

“”"
Created on Wed Feb 27 13:36:00 2019

@author: BERSON
“”"

import urllib.request
import urllib.parse

#根据url发送请求，获取服务器响应文件
def url_open(fullurl, filename):
req = urllib.request.Request(fullurl)
req.add_header(‘User-Agent’, ‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36’)
response = urllib.request.urlopen(fullurl)
html = response.read().decode(‘utf-8’) #对字符串进行转码
return html

#将获取的内容写入本地
def get_page(get, filename):
print(“正在保存” + filename)
with open(filename, “w”, encoding=‘utf-8’) as f:
f.write(get)
print("-" * 30)

#调度器，负责组合处理每个页面url
def tiebaSpider(url, beginpage, endpage):
for page in range(beginpage, endpage + 1):
pn = (page - 1) * 50
filename = “第” + str(page) + “页.html”
fullurl = url + “&pn=” + str(pn)
get = url_open(fullurl, filename)
get_page(get, filename)
print(“谢谢使用”)

if name == ‘main’:
kw = input(“请输入要爬取的贴吧名：”)
beginpage = int(input(“请输入起始页：”))
endpage = int(input(“请输入结束页：”))
url = "https://tieba.baidu.com/f?"
key = urllib.parse.urlencode({“kw” : kw})
fullurl = url + key
tiebaSpider(fullurl, beginpage, endpage)