selenium 实现爬虫

1下载

selenium-server-standalone-2.41.0.jar

chromedriver_win32.zip

 IEDriverServer_x64_2.42.0.zip

    

2设置环境

1)解压chromedriver_win32.zip，把chromedriver.exe拷贝至C:/ selenium/chrome/

2)解压IEDriverServer_x64_2.42.0.zip把IEDriverServer.exe拷贝至C:/ selenium/ie/

3)ie驱动的路径加入至环境变量PATH

 

3 代码示例

 建java工程，添加jar包selenium-server-standalone-2.41.0.jar。

3.1 baidu

import org.openqa.selenium.By;
import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.StaleElementReferenceException;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebDriverException;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.ie.InternetExplorerDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.support.ui.ExpectedCondition;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
    private static void example() 
    {
    	//for firefox
        //WebDriver driver = new FirefoxDriver();

    	//for chrome 
    	System.setProperty("webdriver.chrome.driver", "C:/ selenium/chrome/chromedriver.exe");
    	WebDriver driver = new ChromeDriver();
    	
    	//for IE
    	//WebDriver driver = new InternetExplorerDriver();

    	WebDriverWait w = new WebDriverWait(driver, 10);
    	
        driver.get("http://www.baidu.com/");  
        // 等价于    driver.navigate().to("http://www.baidu.com/");

        w.until(ExpectedConditions.visibilityOfElementLocated(By.id("kw1")));
        w.until(ExpectedConditions.elementToBeClickable(By.id("su1")));
        
        // Find the text input element by its name
        WebElement element = driver.findElement(By.id("kw1"));
        element.sendKeys("liaoxiangui");

        element.submit();  
       //WebDriver会自动的找element对应的form，并提交. 等价于：   
       //driver.findElement(By.id("su1")).click();

        System.out.println("Page title is: " + driver.getTitle());
        
        // baidu's search is rendered dynamically with JavaScript.
        // Wait for the page to load, timeout after 10 seconds
        w.until(new ExpectedCondition<Boolean>() {public Boolean apply(WebDriver d) {return d.getTitle().toLowerCase().startsWith("liaoxiangui");}});
        
        System.out.println("Page title is: " + driver.getTitle());
        
        try{Thread.sleep(10000);}catch(Exception e){}
        
        //Close the browser
        driver.quit();
    }

3.2 一号店

private static void yihaodian()
{
	System.setProperty("webdriver.chrome.driver", "C:/work/research/bijia/selenium/chromedriver_win32/chromedriver.exe");
	
	ChromeOptions options = new ChromeOptions();
	//options.addArguments("--disable-images");
	WebDriver driver = new ChromeDriver(options);
    
	try
	{
	driver.get("http://www.yhd.com/ctg/s2/c21289-0-60761/b/a-s1-v0-p15-price-d0-f0-m1-rt0-pid-mid0-k/#page=1&sort=1");
        
	Boolean first = true;
        int products=0;
        WebDriverWait ww = new WebDriverWait(driver, 10);
        
        while(true)
        {	
            System.out.println("processing filter page:"+driver.getCurrentUrl());
        	
            if(first)
            {
            	first=false;
	            ww.until(ExpectedConditions.presenceOfElementLocated(By.id("startShopping")));		            

	            WebElement s = driver.findElement(By.id("selectProvince"));
	            s.click();
	            s = driver.findElement(By.id("p_13"));
	            s.click();
	            s = driver.findElement(By.id("startShopping"));
	            s.click();
            }
            
            ww.until(ExpectedConditions.presenceOfAllElementsLocatedBy(By.cssSelector("#itemSearchList > li")));
             //窗口最大化。 必须使得那个按钮在屏幕上可见，否则会抛异常?
            //driver.manage().window().maximize();
            final int liCounter = driver.findElements(By.cssSelector("#itemSearchList > li")).size();		        
	        for(int kk=0;kk<liCounter;kk++)
	        {	
	        	try
	        	{
		        	String cssStr = "#itemSearchList > li:nth-of-type("+(kk+1)+")";
		        	WebElement li = driver.findElement(By.cssSelector(cssStr));;
		        	products++;
		        	
		        	String t;

		        	try{
		        	ww.until(ExpectedConditions.visibilityOf(li.findElement(By.cssSelector(".owner > a"))));
		        	t = li.findElement(By.cssSelector(".owner > a")).getAttribute("title");
		        	System.out.println("vendor name="+t);
		        	}
		        	catch(NoSuchElementException e)
		        	{
		        		System.out.println("vendor=自营");
		        	}
		        	
		        	
		        	ww.until(ExpectedConditions.visibilityOf(li.findElement(By.className("electrical_item_box"))));
		        	t = li.findElement(By.className("electrical_item_box")).getAttribute("comproid");
		        	System.out.println("selfid="+t);
		        	
		        	ww.until(ExpectedConditions.visibilityOf(li.findElement(By.cssSelector("div > .search_prod_img > img"))));
		        	t = li.findElement(By.cssSelector("div > .search_prod_img > img")).getAttribute("src");
		        	System.out.println("pic url="+t);
		        	
		        	ww.until(ExpectedConditions.visibilityOf(li.findElement(By.cssSelector(".title > .title"))));
		        	t = li.findElement(By.cssSelector(".title > .title")).getText();
		        	System.out.println("title="+t);
		        	
		        	t = li.findElement(By.cssSelector(".title > .title")).getAttribute("href");
		        	System.out.println("detailed url="+t);

		        	ww.until(ExpectedConditions.visibilityOf(li.findElement(By.className("color_red"))));
		        	t = li.findElement(By.className("color_red")).getText();
		        	System.out.println("price="+t);
	        	}
	        	catch(StaleElementReferenceException ex) //see http://docs.seleniumhq.org/exceptions/stale_element_reference.jsp
	        	{
	        		kk--;
	        		System.out.println("stale element. retry to get it.");
	        	}
	        	//break;
	        }
	        
	        
	        WebElement ne = null;
	        while(true)
	        {
	        	try
	        	{
			        ww.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("#rankOpDiv")));
			        //如果找不到指定的element，selenium的做法是抛异常。
		        	ne = driver.findElement(By.cssSelector("#rankOpDiv .select_page_btn a.next")); //throw NoSuchElementException
		        	ww.until(ExpectedConditions.elementToBeClickable(By.cssSelector("#rankOpDiv a.next")));
		        	ne.click();
	        	}
	        	catch(StaleElementReferenceException ex)
	        	{
	        		System.out.println("retry  going to next page.");
	        		continue;
	        	}
	        	catch(NoSuchElementException e)
	        	{
	        		System.out.println("this category end!");
	        	}
	        	break;
	        }
	        
        	
	        if(ne == null)
	        	break;
        }
        
        System.out.println("get "+products+" products.");
	}
	catch(Exception ex)
	{
		System.out.println(ex);
	}
    
    //Close the browser
    driver.quit();
}