Requests 爬虫贴吧

最新推荐文章于 2021-01-03 05:36:20 发布

weixin_48640346

最新推荐文章于 2021-01-03 05:36:20 发布

阅读量689

点赞数

分类专栏： Python

本文链接：https://blog.csdn.net/weixin_48640346/article/details/109606038

版权

Python 专栏收录该内容

23 篇文章 0 订阅

订阅专栏

import requests

class TiebaSpider():
	def __init__(self,tieba_name):
		self.url_temp = "https://tieba.baidu.com/f?kw= "+tieba_name + "&ie=utf-8&pn={}"
		self.tieba_name = tieba_name
		self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36"}
		
		
	def get_url_list(self):
		'''
		url_list = []
		for i in range(10):
			
			url_list.append(self.url_temp.format(i*50))
		return url_list	
		'''
		return [self.url_temp.format(i*50) for i in range(10)]
	
	
	def parse_url(self,url):
		print(url)
		response = requests.get(url,headers=self.headers)
		return response.content.decode()
	
	
	def save_html(self,html_str,page_num):
		file_path = "{}--第{}页.html".format(self.tieba_name,page_num)
		with open(file_path,"w",encoding='utf-8') as fp:
			fp.write(html_str)
		
	def run(self):
		url_list = self.get_url_list()
		for url in url_list:
			html_str = self.parse_url(url)
			page_num = url_list.index(url)+1
			self.save_html(html_str,page_num)

if __name__ == '__main__':
	tieba_spider = TiebaSpider('晴天')
	tieba_spider.run()