爬取百度热点内容标题链接

import requests

from bs4 import BeautifulSoup

# 发送GET请求
url = 'https://top.baidu.com/board?tab=realtime'
response = requests.get(url)
headers = response.request.headers
html_text = response.text
soup = BeautifulSoup(html_text, 'html.parser')
div_content = soup.find_all("div", style="margin-bottom:20px")
# print(div_content)
# 循环取标签的内容
with open("output.txt", 'w', encoding='utf-8') as file:
    for dic in div_content:
        href_tags = soup.find_all("a", class_='title_dIF3B')
        for h in href_tags:
            # 获取标题+去前后空格
            t_name = h.text.strip()
            t_href = h.get('href')
            result = "标题:", t_name, "链接:", t_href
            result = str(result)
            file.write(result+'\n')