CSDN搜索并爬取数据

import json
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys


def get_cookies():
    web.find_element(By.XPATH,'//*[@id="csdn-toolbar"]/div/div/div[3]/div/div[1]/a').click()
    web.find_element(By.XPATH,'/html/body/div[2]/div/div[2]/div/div[2]/div/div/span[1]').click()
    input()
    cookie=web.get_cookies()
    json_cookie=json.dumps(cookie)
    with open('CSDN的cookie.txt','w') as f:
        f.write(json_cookie)
    print('cookie保存成功！！')

def post_cookie():
    with open('CSDN的cookie.txt','r',encoding='utf-8') as f:
        cookies=json.loads(f.read())

    for cookie in cookies:
        data={
            "domain": ".csdn.net",
            "name": cookie.get("name"),
            'value': cookie.get("value"),
            'expires': '',
            'path': "/",
            'httpOnly': False,
            'HostOnly': False,
            'Secure': False
        }
        web.add_cookie(data)
    web.implicitly_wait(10)
    web.refresh()       # 刷新一下页面，就能成功登录


def post_content():
    content=input('请输入你想要查询的内容：')
    web.find_element(By.XPATH,'//*[@id="toolbar-search-input"]').send_keys(content,Keys.ENTER)
    web.implicitly_wait(10)
    web.switch_to.window(web.window_handles[-1])
    div=web.find_elements(By.XPATH,'//*[@id="app"]/div[2]/div[2]/div[1]/div[2]/div/div')
    web.implicitly_wait(10)
    for a in div:
        title=a.find_element(By.CLASS_NAME,'title.substr').text
        web.implicitly_wait(10)
        href=a.find_element(By.TAG_NAME,'a').get_attribute('href')
        print(f'{title}\n{href}')
        print('--------------------------------')


if __name__ == '__main__':
    web = webdriver.Chrome()
    web.get('https://www.csdn.net/')
    web.maximize_window()
    web.implicitly_wait(10)
    # get_cookies()
    post_cookie()
    web.implicitly_wait(10)
    post_content()

这里的登录要用扫二维码，之后可凭保存下来的cookie直接登录（但时间长了就不行了好像，但可以自行查询资料改一下，应该是可以长期保存的）。