public class baiduNewsPost {
public static void main(String args[])
throws FailingHttpStatusCodeException, MalformedURLException,
IOException, InterruptedException {
final WebClient webclient = new WebClient();
final HtmlPage htmlpage = webclient
.getPage("http://news.baidu.com/");
webclient.getOptions().setCssEnabled(false);
webclient.getOptions().setJavaScriptEnabled(false);
System.out.println(htmlpage.asText());
System.out.println(htmlpage.getTitleText());
final HtmlForm form = htmlpage.getFormByName("fbaidu");
final HtmlSubmitInput button = form.getInputByValue("百度一下");
final HtmlTextInput textField = form.getInputByName("word");
String keyword = "***";
textField.setValueAttribute(keyword);
HtmlPage page = button.click();
System.out.println(page);
textField.setValueAttribute(keyword);
HashSet<String> ts = new HashSet<String>();
int pagenum = 1;
while (page != null && pagenum < 38) {
java.util.List<HtmlAnchor> achList = page.getAnchors();
for (HtmlAnchor ach : achList) {
String url = ach.getHrefAttribute();
String s = url.substring(0, 1);
String regex = ".*?baidu.*?";
Pattern p = Pattern.compile(regex);
Matcher m = p.matcher(url);
if (s.equals("/") || s.equals("j") || url.length() > 100
|| m.find()) {
continue;
}
System.out.println(url);
ts.add(url);
}
HtmlElement elepage = page.getHtmlElementById("page");
HtmlElement nextpage = null;
//获取下一页
if(pagenum == 1){
nextpage = (HtmlElement) elepage.getByXPath("//a[@class='n']")
.get(0);
}else if(elepage.getByXPath("//a[@class='n']").size() == 2){
nextpage = (HtmlElement) elepage.getByXPath("//a[@class='n']")
.get(1);
}
if(nextpage != null){
//点击下一页
page = nextpage.click();
}else {
page = null;
continue;
}
System.out.println("pagenum :" + pagenum + nextpage.asText());
pagenum ++;
//生成随机睡眠时间,防止被百度屏蔽
double time = Math.random()*45000;
int sleeptime = (int)time + 30000;
Thread.sleep(sleeptime);
}
webclient.closeAllWindows();
}
}
使用htmlunit 获取百度新闻搜索结果的url
最新推荐文章于 2024-06-13 01:03:00 发布