htmlUnit基础操作

@SuppressWarnings("resource")

public HtmlPage getpageInfo(String url) {

WebClient wc = null;

HtmlPage page = null;

try {

wc = new WebClient(BrowserVersion.CHROME, "10.27.199.6", 3128);

wc.getOptions().setThrowExceptionOnFailingStatusCode(true);

wc.getOptions().setThrowExceptionOnScriptError(false);

wc.getOptions().setCssEnabled(false);

wc.getOptions().setJavaScriptEnabled(false);

wc.getOptions().setUseInsecureSSL(true);

wc.getOptions().setTimeout(10 * 1000);

wc.setAjaxController(new NicelyResynchronizingAjaxController());

wc.setRefreshHandler(new NiceRefreshHandler(3));

wc.setIncorrectnessListener(new IncorrectnessListener() {

@Override

public void notify(String message, Object origin) {

// do nothing.

}

});

wc.getCache().setMaxSize(0);

// wap 评论点赞接口

WebRequest wrWap = generateRequest(url, null, null);

page = wc.getPage(wrWap);

} catch (Exception e) {

LOGGER.error("抓取数据异常", e);

}finally {

wc.getCurrentWindow().getJobManager().removeAllJobs();

wc.close();

System.gc();

}

return page;

}

private static WebRequest generateRequest(String url, String cookieStr,

String referer) throws MalformedURLException {

WebRequest request = new WebRequest(new URL(url));

request.setAdditionalHeader(

"User-Agent",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36");

request.setAdditionalHeader("Accept",

"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,* ");

request.setAdditionalHeader("Accept-Encoding", "gzip, deflate, sdch");

request.setAdditionalHeader("Accept-Language", "zh-CN,zh;q=0.8");

request.setAdditionalHeader("Connection", "keep-alive");

if (cookieStr != null) {

request.setAdditionalHeader("Cookie", cookieStr);

}

if (referer != null) {

request.setAdditionalHeader("Referer", referer);

}

request.setHttpMethod(HttpMethod.GET);

return request;

}