使用htmlunit 获取百度新闻搜索结果的url

public class baiduNewsPost {
	public static void main(String args[])
			throws FailingHttpStatusCodeException, MalformedURLException,
			IOException, InterruptedException {
		final WebClient webclient = new WebClient();
		final HtmlPage htmlpage = webclient
				.getPage("http://news.baidu.com/");

		webclient.getOptions().setCssEnabled(false);
		webclient.getOptions().setJavaScriptEnabled(false);
		 System.out.println(htmlpage.asText());
		 System.out.println(htmlpage.getTitleText());
		final HtmlForm form = htmlpage.getFormByName("fbaidu");
		final HtmlSubmitInput button = form.getInputByValue("百度一下");
		final HtmlTextInput textField = form.getInputByName("word");
		String keyword = "***";
		textField.setValueAttribute(keyword);
		HtmlPage page = button.click();
		System.out.println(page);
		
		textField.setValueAttribute(keyword);
		HashSet<String> ts = new HashSet<String>();
                int pagenum = 1;
		while (page != null && pagenum < 38) {
			java.util.List<HtmlAnchor> achList = page.getAnchors();

			for (HtmlAnchor ach : achList) {
				String url = ach.getHrefAttribute();
				String s = url.substring(0, 1);
				String regex = ".*?baidu.*?";
				Pattern p = Pattern.compile(regex);
				Matcher m = p.matcher(url);
				if (s.equals("/") || s.equals("j") || url.length() > 100
						|| m.find()) {
					continue;
				}
				System.out.println(url);
				ts.add(url);
			}
			HtmlElement elepage = page.getHtmlElementById("page");
			HtmlElement nextpage = null;
			//获取下一页
			if(pagenum == 1){
				nextpage = (HtmlElement) elepage.getByXPath("//a[@class='n']")
					.get(0);
			}else if(elepage.getByXPath("//a[@class='n']").size() == 2){
				nextpage = (HtmlElement) elepage.getByXPath("//a[@class='n']")
						.get(1);
			}
			if(nextpage != null){
				//点击下一页
				page = nextpage.click();
			}else {
				page = null;
				continue;
			}
			System.out.println("pagenum :" + pagenum + nextpage.asText());
			pagenum ++;
			
			//生成随机睡眠时间，防止被百度屏蔽
			double time = Math.random()*45000;
			int sleeptime = (int)time + 30000;
			Thread.sleep(sleeptime);
		}
		webclient.closeAllWindows();
	}
}