#!/usr/bin/env python
# -*- coding:utf-8 -*-
import urllib.request
import random
'''
爬取百度贴吧
参数:贴吧名称、起始页、结束页
Python3.7.0
'''
def getUserAgent():
'''
作用:随机获取浏览器的userAgent
'''
# 模拟浏览器的请求,反爬虫的第一步
ua_list = [
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv2.0.1) Gecko/20100101 Firefox/4.0.1",
"Mozilla/5.0 (Windows NT 6.1; rv2.0.1) Gecko/20100101 Firefox/4.0.1",
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
"Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"
]
# 在User-Agent列表里面随机选择一个User-Agent
user_agent = random.choice(ua_list)
return user_agent
def loadPage(url, filename):
'''
作用:根据url发送请求,获取服务器响应文本
url:需要爬取的url地址
'''
print("正在下载" + filename)
request = urllib.request.Request(url)
request.add_header("User-Agent", getUserAgent())
return urllib.request.urlopen(request).read()
def writePage(html, filename):
'''
作用:将html内容写入到本地
html:服务器响应文件的内容
'''
print("正在保存" + filename)
# 文件写入 with 不需要做文件打开关闭操作 注:这里是坑,需要用wb+的方式写入文件
with open(filename, 'wb+') as f:
f.write(html)
print('-' * 30)
def tiebaSpider(url, beginPage, endPage):
'''
作用:贴吧爬虫调度器,负责组合处理每个页面的url
url:贴吧url的前部分
beginPage: 起始页
endPage:结束页
'''
for page in range(beginPage, endPage + 1):
pn = (page - 1) * 50
filename = '第' + str(page) + '页.html'
fullurl = url + "&pn=" + str(pn)
html = loadPage(fullurl, filename)
writePage(html, filename)
print("谢谢使用")
if __name__ == "__main__": # 当.py文件被直接运行时,if __name__ == '__main__'之下的代码块将被运行;当.py文件以模块形式被导入时,if __name__ == '__main__'之下的代码块不被运行。
kw = input("请输入需要爬取的贴吧名: ")
beginPage = int(input("输入起始页: "))
endPage = int(input("请输入结束页: "))
url = "https://tieba.baidu.com/f?"
key = urllib.parse.quote("kw=" + kw) # 此处与Python2(是坑) 不同,值必须是字符串的类型
# print(key)
fullurl = url + key
# print(fullurl)
tiebaSpider(fullurl, beginPage, endPage)
注:转url编码的时候Python2与Python3 的写法不同
formdata = {
"page_limit":"20",
"page_start":"20"
}
# 转成url的编码
data = urllib.parse.urlencode(formdata).encode("utf-8") # Python3.7.0
data = urllib.urlencode(formdata) # Python2