新手第一次写博客,仿照老师给的爬虫结构写了一个
用python写的爬虫,爬取了热搜榜的内容
源码:
# -*- coding: utf-8 -*-
"""
Created on Sun Mar 5 09:38:08 2023
@author: DELL
"""
url = 'http://top.baidu.com/buzz?b=1&fr=topindex'
def getSoup(Url):
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36'
} # 设置请求头
proxy = {'http': 'http://101.4.136.34:82'}# 设置代理 http-协议类型 101.4.136.34-代理ip 82-代理端口
urls=requests.get(url,headers=headers)
r = requests.get(url, headers=headers, timeout=30)
r.encoding = r.apparent_encoding# 获取网页的编码格式
context = r.text # 获取HTML网页
soup = BeautifulSoup(r.text,'html.parser') # 解析网页
return soup
def getContext():
soup = getSoup(url)
# 获取<div class='category-wrap_iQLoo horizontal_1eKyQ'></div>所有标签项
info_clear_all = soup.find_all('div', class_='category-wrap_iQLoo horizontal_1eKyQ')
for a in info_clear_all:
# 获取标题
label_a_title = a.find('div', class_='c-single-text-ellipsis') # 获取标题所在的a标签
title = label_a_title.text.replace(' ', '').strip().replace('\n', '') # 获取标题
print('标题:'+title)
# 获取热搜指数
hot_index = a.find('div', class_='hot-index_1Bl1a')
hot_index_num = hot_index.text.replace(' ', '').strip().replace('\n', '')
print('热搜指数:'+hot_index_num)
if __name__ == "__main__":
getContext()
结果如下:
![](https://i-blog.csdnimg.cn/blog_migrate/77eca8c774b66b16980dd2afeff3e988.png)
![](https://i-blog.csdnimg.cn/blog_migrate/a91fb3f836a8308aa06e9916ab6d35c3.png)
![](https://i-blog.csdnimg.cn/blog_migrate/b96f71b3d4977bffdc2a6ca47b095bf5.png)