htmlunit实例

最新推荐文章于 2021-06-05 06:13:13 发布

m313837122

最新推荐文章于 2021-06-05 06:13:13 发布

阅读量183

点赞数

分类专栏：应用技术文章标签： java

本文链接：https://blog.csdn.net/m313837122/article/details/84620185

版权

应用技术专栏收录该内容

13 篇文章 0 订阅

订阅专栏

htmlunit 是一款开源的java 页面分析工具，读取页面后，可以有效的使用htmlunit分析页面上的内容。项目可以模拟浏览器运行，被誉为java浏览器的开源实现。这个没有界面的浏览器，运行速度也是非常迅速的。
相关文件下载地址：

http://sourceforge.net/projects/htmlunit/files/

http://jaist.dl.sourceforge.net/project/htmlunit/htmlunit/2.15/htmlunit-2.15-bin.zip

我的需求是使用百度的高级新闻搜索，抓取指定站点新闻
手动搜索的设置如图所示

 
         package 
         com.html580; 
        
         import 
         java.io.IOException; 
        
         import 
         java.net.MalformedURLException; 
        
         import 
         java.util.List; 
        
         import 
         com.gargoylesoftware.htmlunit.ElementNotFoundException; 
        
         import 
         com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException; 
        
         import 
         com.gargoylesoftware.htmlunit.WebClient; 
        
         import 
         com.gargoylesoftware.htmlunit.html.HtmlForm; 
        
         import 
         com.gargoylesoftware.htmlunit.html.HtmlHiddenInput; 
        
         import 
         com.gargoylesoftware.htmlunit.html.HtmlPage; 
        
         import 
         com.gargoylesoftware.htmlunit.html.HtmlRadioButtonInput; 
        
         import 
         com.gargoylesoftware.htmlunit.html.HtmlSelect; 
        
         import 
         com.gargoylesoftware.htmlunit.html.HtmlSubmitInput; 
        
         import 
         com.gargoylesoftware.htmlunit.html.HtmlTextInput; 
        
         /** 
        
         * @description 抓取百度搜索结果 
        
         * @author html580 
        
         * @site http://www.html580.com 
        
         * @version 2014-7-23 
        
         */ 
        
         public 
         class 
         BaiduSpider { 
        
         public 
         static 
         void 
         main(String[] args) { 
        
         try 
         { 
        
         final 
         WebClient webclient =  
         new 
         WebClient(); 
        
         final 
         HtmlPage htmlpage = webclient.getPage( 
         "http://www.baidu.com/gaoji/advanced.html" 
         ); 
        
         //搜索按钮 
        
         final 
         HtmlForm form = htmlpage.getFormByName( 
         "f1" 
         ); 
        
         final 
         HtmlSubmitInput button = form.getInputByValue( 
         "百度一下" 
         ); 
        
         //搜索结果-关键词 
        
         final 
         HtmlTextInput textField = form.getInputByName( 
         "q1" 
         ); 
        
         textField.setValueAttribute( 
         "HTML我帮您" 
         ); 
        
         //分页条数 
        
         final 
         HtmlSelect htmlSelet=form.getSelectByName( 
         "rn" 
         ); 
        
         htmlSelet.setDefaultValue( 
         "10" 
         ); 
        
         //网页的时间 
        
         final 
         HtmlSelect htmlSeletlm=form.getSelectByName( 
         "rn" 
         ); 
        
         htmlSeletlm.setDefaultValue( 
         "0" 
         ); 
        
         //语言 
        
         final 
         List<HtmlRadioButtonInput> radioButtonCts = form.getRadioButtonsByName( 
         "ct" 
         ); 
        
         radioButtonCts.get( 
         0 
         ).setChecked( 
         true 
         ); 
        
         radioButtonCts.get( 
         1 
         ).setChecked( 
         false 
         ); 
        
         radioButtonCts.get( 
         2 
         ).setChecked( 
         false 
         ); 
        
         //文档格式 
        
         final 
         HtmlSelect htmlSeletft=form.getSelectByName( 
         "ft" 
         ); 
        
         htmlSeletft.setDefaultValue( 
         "" 
         ); 
        
         //关键词位置 
        
         final 
         List<HtmlRadioButtonInput> radioButtonq5s = form.getRadioButtonsByName( 
         "q5" 
         ); 
        
         radioButtonq5s.get( 
         0 
         ).setChecked( 
         true 
         ); 
        
         radioButtonq5s.get( 
         1 
         ).setChecked( 
         false 
         ); 
        
         radioButtonq5s.get( 
         2 
         ).setChecked( 
         false 
         ); 
        
         //站内搜索      限定要搜索指定的网站 
        
         final 
         HtmlTextInput htmlTextInputq6 = form.getInputByName( 
         "q6" 
         ); 
        
         htmlTextInputq6.setDefaultValue( 
         "html580.com" 
         ); 
        
         //隐藏值 
        
         final 
         HtmlHiddenInput hiddenInputtn = form.getInputByName( 
         "tn" 
         ); 
        
         hiddenInputtn.setDefaultValue( 
         "baiduadv" 
         ); 
        
         final 
         HtmlPage page2 = button.click(); 
        
         String result = page2.asXml(); 
        
         System.out.println(result); 
        
         webclient.closeAllWindows(); 
        
         }  
         catch 
         (FailingHttpStatusCodeException e) { 
        
         e.printStackTrace(); 
        
         }  
         catch 
         (MalformedURLException e) { 
        
         e.printStackTrace(); 
        
         }  
         catch 
         (ElementNotFoundException e) { 
        
         e.printStackTrace(); 
        
         }  
         catch 
         (IOException e) { 
        
         e.printStackTrace(); 
        
         } 
        
         } 
        
         }