import requests, json, re, random,time
from bs4 import BeautifulSoup
from selenium import webdriver
# from selenium.webdriver.common.by import By
from lxml import etree
import pandas as pd
"""
使用selenium进行模拟登陆
"""
driver = webdriver.Chrome('./chromedriver')
driver.get('https://news.qq.com/')
# 将窗口调整最大
driver.maximize_window()
# 休息5s
time.sleep(5)
current_window_1 = driver.current_window_handle
# print(current_window_1)
js = "document.documentElement.scrollTop=500000"
for i in range(15):
driver.execute_script(js)
time.sleep(3)
soup = BeautifulSoup(driver.page_source,"html.parser")
titles = []
urls = []
for info in soup.find(lambda tag: tag.get('class')==['list']).find_all(lambda tag: tag.get('class')==['detail']):
print(info.a.text, '\t', info.a["href"], '\n')
titles.append(info.a.text)
urls.append(info.a["href"])
import pandas as pd
tencent_news = pd.DataFrame([titles, urls]).T
tencent_news.rename(columns={0:'title',1:'url'},inplace=True)
print(tencent_news)
代码运行结果: