用python实现爬取CSDN热门评论URL并存入redis

一、配置webdriver

下载谷歌浏览器驱动，并配置好

import time
import random
from PIL import Image
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
if __name__ == '__main__':
    options = webdriver.ChromeOptions()
    options.binary_location = r'C:UsershhhAppDataLocalGoogleChromeApplication谷歌浏览器.exe'
    # driver=webdriver.Chrome(executable_path=r'D:eChromechromedriverchromedriver.exe')
    driver = webdriver.Chrome(options=options)
    #以java模块为例
    driver.get('https://www.csdn.net/nav/java')
    for i in range(1,20):
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
        time.sleep(2)

二、获取URL

from bs4 import BeautifulSoup
from lxml import etre