我们使用phantomjs抓取页面,可能需要登录,但是又想要登录一次,以后每次都要输入密码,就需要设置cookie。
phantomjs设置cookie方法:
以ChromeDriver为例:
package com.tomtop.big.data.crawler.ebay;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.Date;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.concurrent.TimeUnit;
import org.junit.Test;
import org.openqa.selenium.By;
import org.openqa.selenium.By.ByTagName;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.client.RestTemplate;
import com.tomtop.big.data.base.utils.TomtopDateUtil;
import com.tomtop.big.data.crawler.browser.amazon.AmazonHtmlunitBrowser;
import com.tomtop.big.data.crawler.proxy.amazon.ReviewForeignProxyHelp;
//@RunWith(SpringJUnit4ClassRunner.class)
//@ContextConfiguration(locations = { "classpath*:etc/spring/*" })
@SuppressWarnings("all")
public class Phantomjstest {
@Autowired
protected ReviewForeignProxyHelp foreignProxyHelp;
private AmazonHtmlunitBrowser htmlunitBrowser;
@Autowired
private RestTemplate template;
Set<Cookie> set;
@Test
public void f1() throws Exception {
while(true) {
phantomjsTest();
Thread.sleep(3000);
}
}
// @Test
public void phantomjsTest() throws Exception {
System.setProperty("webdriver.chrome.driver",
"D:\\phantomjs\\phantomjs-2.1.1-windows\\bin\\chromedriver.exe");
ChromeDriver driver = new ChromeDriver();
driver.get("https://www.ebay.com/sh/ovw");
if(set != null) {
for (Cookie cookie : set) {
driver.manage().addCookie(cookie);
}
}
driver.get("https://www.ebay.com/sh/ovw");
driver.manage().timeouts().implicitlyWait(1, TimeUnit.SECONDS);
String currentUrl = driver.getCurrentUrl();
if (currentUrl.contains("signin")) {
System.err.println("进入登陆页面");
// 登陆页面
WebElement useridEle = driver.findElementById("userid");
WebElement passEle = driver.findElementById("pass");
WebElement btnEle = driver.findElementById("sgnBt");
useridEle.sendKeys("tomtop_game");
passEle.sendKeys("runningMen*2012");
btnEle.submit();
set = driver.manage().getCookies();
for (Cookie cookie : set) {
System.err.println(cookie.getDomain());
System.err.println(cookie.getName());
System.err.println(cookie.getPath());
System.err.println(cookie.getValue());
if(cookie.getExpiry() != null) {
System.err.println(TomtopDateUtil.getFormat2().format(cookie.getExpiry()));
}
System.err.println("=================================");
}
}
Thread.sleep(1000L);
String windowHandle = driver.getWindowHandle();
driver.switchTo().window(windowHandle);
WebElement positiveEle = driver
.findElementByClassName("feedback-positive");
By by = new ByTagName("a");
WebElement element = positiveEle.findElement(by);
System.err.println(element.getText());
driver.close();
driver.quit();
}
}
关键的地方在于:你要请求两次,先请求一次,再设置cookie,然后再请求一次。
driver.get("https://www.ebay.com/sh/ovw");
if(set != null) {
for (Cookie cookie : set) {
driver.manage().addCookie(cookie);
}
}
driver.get("https://www.ebay.com/sh/ovw");