Datewhale组队学习-爬虫task04学习笔记

import requests, json, re, random,time
from bs4 import BeautifulSoup
from selenium import webdriver
# from selenium.webdriver.common.by import By
from lxml import etree
import pandas as pd


"""
使用selenium进行模拟登陆
"""

driver = webdriver.Chrome('./chromedriver')
driver.get('https://news.qq.com/')
# 将窗口调整最大
driver.maximize_window()
# 休息5s
time.sleep(5)
current_window_1 = driver.current_window_handle
# print(current_window_1)

js = "document.documentElement.scrollTop=500000"
for i in range(15):
    driver.execute_script(js)
    time.sleep(3)


soup = BeautifulSoup(driver.page_source,"html.parser")

titles = []
urls = []
for info in soup.find(lambda tag: tag.get('class')==['list']).find_all(lambda tag: tag.get('class')==['detail']):
    print(info.a.text, '\t', info.a["href"], '\n')
    titles.append(info.a.text)
    urls.append(info.a["href"])

import pandas as pd

tencent_news = pd.DataFrame([titles, urls]).T
tencent_news.rename(columns={0:'title',1:'url'},inplace=True)
print(tencent_news)

代码运行结果：