通过selenium chromedriver爬取评论
模拟浏览器访问来获取数据
1下载谷歌浏览器引擎 chromedriver
2maven 依赖
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>3.4.0</version>
</dependency>
3 爬取代码
package a.a;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
/**
* Hello world!
*
*/
public class App
{
public static void main( String[] args )
{
//chromedriver存放地址;
System.setProperty("webdriver.chrome.driver", "D:/chromedriver_win32/chromedriver.exe");
//调用chrome
WebDriver driver = new ChromeDriver();
//调整高度
//((ChromeDriver) driver).executeScript("window.scrollTo(0, document.body.scrollHeight);");
//获取网址
((ChromeDriver) driver).get("https://www.baidu.com/s?tn=02003390_19_hao_pg&ie=utf-8&wd=%E8%AE%A1%E7%AE%97%E5%99%A8");
//根据class获取内容
WebElement element = driver.findElement(By.className( "BOJBWh64"));
String content=element.getText();
System.out.println(content);
//关闭浏览器
driver.close();
}
}
如有错误欢迎指正