selenium 爬取数据案例

依赖

        <dependency>
            <groupId>org.seleniumhq.selenium</groupId>
            <artifactId>selenium-chrome-driver</artifactId>
            <version>3.141.59</version>
        </dependency>

        <dependency>
            <groupId>org.seleniumhq.selenium</groupId>
            <artifactId>selenium-java</artifactId>
            <version>3.141.59</version>
        </dependency>

代码

package com.example.bootmagic.selenium;

import org.openqa.selenium.*;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;

import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;

public class MainClass {
    // 官网
    static String URL_SELENIUM = "https://www.selenium.dev/";
    static String URL_JD = "https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&wq=%E6%89%8B%E6%9C%BA&pvid=e6619f8a87004312989b6ca7462adc92";

    public static void main(String[] args) throws InterruptedException {
        System.setProperty("webdriver.chrome.driver", "E:\\WebDriver\\bin\\chromedriver.exe");
        ChromeOptions options = new ChromeOptions();
//         options.addArguments("--headless");
//         options.addArguments("--disable-gpu");
        options.setPageLoadStrategy(PageLoadStrategy.NORMAL);
        WebDriver driver = new ChromeDriver(options);
        WebDriver.Options manage = driver.manage();
        manage.timeouts().setScriptTimeout(3, TimeUnit.SECONDS);
        manage.window().fullscreen();
        manage.window().maximize();
        driver.get(URL_JD);

        List<WebElement> pages = driver.findElements(By.className("p-skip"));
        WebElement b = pages.get(0).findElement(By.tagName("b"));
        Integer total = Integer.parseInt(b.getText());
        System.out.println("总页数" + total);

        for (int j = 0; j < total; j++) {
            fixedData(driver);
            Thread.sleep(getRandomSecond());// 随机歇息几秒后再翻页
            driver.findElement(By.className("pn-next")).click();
        }
        driver.quit();
    }

    // 随机获取秒数
    private static int getRandomSecond() {
        Random random = new Random();
        return (random.nextInt(1) + 3) * 1000;
    }

    private static void fixedData(WebDriver driver) throws InterruptedException {
        fixedScrollPage((JavascriptExecutor) driver);
        Thread.sleep(getRandomSecond());
        List<WebElement> elements = driver.findElements(By.className("gl-i-wrap"));
        int i = 0;
        for (WebElement element : elements) {
            WebElement div = element.findElement(By.tagName("img"));
            System.out.println("************************************************" + (i++));
            System.out.println(div.getAttribute("src"));
            System.out.println(element.getText());
        }
    }

    private static void fixedScrollPage(JavascriptExecutor driver) throws InterruptedException {
        JavascriptExecutor jsExecutor = driver;

        Integer start = 0;
        String increment = "var q=document.documentElement.scrollTop=";
        for (int i = 0; i < 16; i++) {
            String value = increment + start;
            jsExecutor.executeScript(value);
            Thread.sleep(1000);
            start += 500;
        }

    }
}