java+Selenium实现知网数据采集

一个简单的实现知网文章题目和作者等自动化项目。

import java.util.ArrayList;

import org.openqa.selenium.By;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;

public class autozhiwang {

	public static void main(String[] args) throws InterruptedException {

		String search = "云计算";

		// 1.把chrome浏览器驱动加载到系统变量中
		System.setProperty("webdriver.chrome.driver", "src/com/kingdee/web/chromedriver.exe");
		// 2.创建chrome驱动，创建chrome驱动变量
		ChromeDriver driver = new ChromeDriver();
		// 3.访问知网
		driver.get("https://www.cnki.net/");
		Thread.sleep(500);
		// 4.搜索云计算
		driver.findElement(By.id("txt_SearchText")).click();
		driver.findElement(By.id("txt_SearchText")).clear();
		driver.findElement(By.id("txt_SearchText")).sendKeys(search);
		// 5.点击搜索
		WebElement element1 = driver.findElement(By.xpath("/html/body/div[2]/div[2]/div/div[1]/input[2]"));
		element1.click();
		Thread.sleep(500);

		// 6.切换到搜索结果所在的frame
		WebElement iframeResult = driver.findElement(By.id("iframeResult"));
		driver.switchTo().frame(iframeResult);
		// 7.输出搜索到的结果
		for (int i = 0; i < 4000; i++) {
			WebElement element3 = driver.findElement(By.xpath("//*[@id=\"ctl00\"]/table/tbody/tr[2]/td/table"));
			ArrayList<WebElement> Value = (ArrayList<WebElement>) element3.findElements(By.tagName("tr"));
			Value.remove(0);
			for (WebElement webElement : Value) {
				if (webElement.findElement(By.tagName("a")) != null) {
					WebElement as = webElement.findElement(By.tagName("a"));
					System.out.print(as.getText());
					System.out.print("         ");
				}
				if (webElement.findElement(By.className("author_flag")) != null) {
					WebElement as = webElement.findElement(By.className("author_flag"));
					System.out.println(as.getText());
				}
			}
			// 8.分情况点击“下一页”
			if (i < 1) {
				WebElement element4 = driver
						.findElement(By.xpath("//*[@id=\"ctl00\"]/table/tbody/tr[3]/td/table/tbody/tr/td/div/a[9]"));
				element4.click();
			} else {
				WebElement element4 = driver
						.findElement(By.xpath("//*[@id=\"ctl00\"]/table/tbody/tr[3]/td/table/tbody/tr/td/div/a[11]"));
				element4.click();

			}
		}
		// 9.关闭驱动
		driver.quit();
	}
}