java webdriver爬虫_使用java+selenium+maven时间简单的爬虫

package com.WebSelenium;

import java.util.List;

import java.util.Set;

import org.openqa.selenium.By;

import org.openqa.selenium.WebDriver;

import org.openqa.selenium.WebElement;

import org.openqa.selenium.chrome.ChromeDriver;

import

org.openqa.selenium.support.ui.ExpectedConditions;

import org.openqa.selenium.support.ui.WebDriverWait;

public class Test {

private static WebDriver driver;

public static void main(String[] args) throws

InterruptedException {

driver = new ChromeDriver();

landLagouAndFetchDate();

}

private static void landLagouAndFetchDate() throws

InterruptedException {

driver.get("http://www.baidu.com");

driver.manage().window().maximize();

driver.findElement(By.id("kw")).sendKeys("拉钩");

driver.findElement(By.id("su")).click();

String currentHandle = driver.getWindowHandle();

driver.findElement(By.partialLinkText("-专业的互联网招聘平台_找工作_招聘_人才网_求职")).click();

Set handles = driver.getWindowHandles();

for (String handle : handles) {

if (!handle.equals(currentHandle)) {

driver.switchTo().window(handle);

}

}

try {

getElement(By.partialLinkText("全国站")).click();;

} catch (Throwable e) {

// TODO: handle exception

landLagouAndFetchDate();

}

//输入查询数据

getElement(By.id("search_input")).sendKeys("自动化测试工程师");

getElement(By.id("search_button")).click();

List webElements = geElements(By.className("default_list"));

for (WebElement webElement:webElements) {

System.out.println(webElement.getText());

System.out.println("--------------------------");

}

while(true) {

WebElement nextPage=

getElement(By.className("pager_next"));

if (nextPage!=null && nextPage.isEnabled()) {

nextPage.click();

Thread.sleep(5000);

List webElements2 =

geElements(By.className("default_list"));

for (WebElement webElement:webElements2) {

System.out.println(webElement.getText());

System.out.println("-------------------");

}

}else {

break;

}

}

}

private static List geElements(By locator){

WebDriverWait wait = new WebDriverWait(driver, 30);

try {

List webElements =

wait.until(ExpectedConditions.presenceOfAllElementsLocatedBy(locator));

return webElements;

} catch (Exception e) {

System.out.println("定位元素超时了");

}

return null;

}

public static WebElement getElement(By locator) {

WebDriverWait wait = new WebDriverWait(driver, 30);

try {

WebElement webElement =

wait.until(ExpectedConditions.presenceOfElementLocated(locator));

return webElement;

} catch (Exception e) {

// TODO: handle exceptiol

System.out.println("定位元素超时了");

}

return null;

}

}