使用bs4从某站爬题

import requests
from bs4 import BeautifulSoup

def process_selection(selection):
    label = selection.find_next("div", class_="label")
    content = selection.find_next("div", class_="content")
    print(label.text + '、' + content.text)
    return content,content.text

# 设置请求头，包含登录所需的信息
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
    'Cookie': '',  # 替换成登录后的Cookie值
}

# 使用请求头发送请求
#data_url=""
if 'data_url' in vars():
    response = requests.get(data_url, headers=headers)
    # 检查是否成功获取数据
    if response.status_code == 200:
        data = response.text
        with open('c:/Users/lxc11/Desktop/题/32test.html', 'w', encoding='utf-8') as file:
            file.write(data)
    else:
        print("Failed to retrieve data")
else:
    print("从文件获取题目信息")
    with open('c:/Users/lxc11/Desktop/题/31test.html', 'r', encoding='utf-8') as file:
        data = file.read()

soup = BeautifulSoup(data, "html.parser")

# 在这里使用BeautifulSoup提取您所需的数据
# 例如：提取标题
title = soup.title.text

question_content = soup.find("div", class_="tw-flex tw-items-center")

num = 1
while question_content:
    try:
        question_content = question_content.find_next("div", class_="commonPaperHtml")
        print(str(num) + '.' +question_content.text)
        num += 1

        selection, contents = process_selection(question_content)
        selection, contents = process_selection(selection)
        if contents != "错误" and contents != "FALSE":
            selection, contents = process_selection(selection)
            selection, contents = process_selection(selection)

        question_content = selection.find_next("span", class_="tw-text-green-500")
        print("答案：" + question_content.text)
    except Exception:
        print("扫描结束")