爬取凤凰网站财经类的新闻，函数时编程，可全部实现，由于内容量大，需要时间太长，服务器会禁止，为了防止，可以将time.sleep()设置的时间长点...-CSDN博客

import requests
from selenium import webdriver
import time
def grasp(urlT):
    driver = webdriver.Chrome(r'C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe') #自动化测试程序工具本地所在地
    resAll = []         #用于存储单条数据
    rest = {}           #用于存储单个数据
    urls = []
    res=requests.get(urlT)
    for i in range(0,29):

        print(f'第{i+1}条新闻开始')
        print(res.json()['data'][i]['title'])
        try:
            print(res.json()['data'][i]['newsTime'])
        except:
            print('None')
        print(res.json()['data'][i]['source'])
        rest['title']=res.json()['data'][i]['title']
        try:
            rest['newsTime'] = res.json()['data'][i]['newsTime']
        except:
            rest['newsTime'] = 'None'
        rest['source'] = res.json()['data'][i]['source']
        url = res.json()['data'][i]['url']
        rest['url'] = res.json()['data'][i]['url']

        try:

            driver.get(url)
            time.sleep(4)
            contend = driver.find_element_by_class_name('text-3zQ3cZD4').text
            rest['contend'] = str(contend)
            print(f'第{i+1}条新闻结束')
            time.sleep(6)
        except:
            rest['contend'] = '嵌套'
            time.sleep(6)
            l = driver.find_elements_by_xpath("//p[@class='text-3YbAxaNR']")  #获取连接个数
            s = driver.find_elements_by_xpath("//p[@class='text-3YbAxaNR']/a") #获取当前页面所有链接
            for j in range(0,len(l)):
                ss = s[j].get_attribute('href')
                print(type(ss))
                try:
                    urls.append(str(str(ss).split()).replace('"','').replace("'","").replace('[','').replace(']','')) #将链接转化为可以存储的字符串
                    print(urls)
                except:
                    print(driver.find_element_by_class_name('topic-3bY8Hw-9').text) #输出标题
        resAll.append(rest)
        with open('./news.txt', 'a+', encoding='utf-8') as f:
                try:
                    f.write(''.join(resAll[i].values())+'\n')
                except:
                    print('写入失败')


    resAll.clear()
    print(urls)
    for k in range(0,len(urls)):
        try:
            driver.get(urls[k])
            # time.sleep(3)
            rest['title1'] = driver.find_element_by_class_name('topic-3bY8Hw-9').text
            rest['source1'] = driver.find_element_by_class_name('source-2pXi2vGI').text
            rest['newsTime1'] = driver.find_element_by_xpath('//p[@class="time-hm3v7ddj"]/span').text
            rest['contend1'] = driver.find_element_by_class_name('text-3zQ3cZD4').text
            resAll.append(rest)
            time.sleep(4)
            with open('./news.txt', 'a+', encoding='utf-8') as f:
                time.sleep(5)
                f.write(''.join(resAll[k].values()) + '\n')
        except:
            print('内容太多,服务器禁止')


url = "https://shankapi.ifeng.com/spring/finance/index/newInfoIndex/75219" #凤凰网财经的api
t = grasp(url)