使用jsoup爬虫抓取页面

package com.chongdong.log.test; import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.jsoup.Connection; import org.jsoup.Connection.Method; import org.jsoup.Jsoup; import org.jsoup.helper.HttpConnection.Response; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; import org.junit.Test; /** * * 类名称：JsoupTest * 类描述： jsoup 抓取 mitsuku聊天 信息* 创建人：zk * 创建时间：2015-7-20 下午3:52:06 * 修改人：zk * 修改时间：2015-7-20 下午3:52:06 * 修改备注： * 开发进度：* @version 1.0 * */ public class JsoupTest { public static void postMitSuKu (){ Map<String, String> map = new HashMap<String, String>(); /** 表单 提交的参数 * input:how old are you botid:9fa364f2fe345a10 custid:c04f62ad1e044059 faq : http://www.pandorabots.com/botmaster/en/faq#h1 * **/ map.put( "input" , "hi" ); map.put( "botid" , "9fa364f2fe345a10" ); //开启 可在 跳转到聊天框界面 获取 标示 map.put( "custid" , "c04f62ad1e044059" ); //当一个空请求 即可获取 相当于 标示 Connection conn = Jsoup.connect( "http://fiddle.pandorabots.com/pandora/talk-xml" ); /* conn.header("(Request-Line)", "POST /cgi-bin/login?lang=zh_CN HTTP/1.1");*/ conn.header( "Accept" , "*/*" ); conn.header( "Accept-Encoding" , "gzip,deflate,sdch" ); conn.header( "Accept-Language" , "zh-CN,zh;q=0.8" ); /* conn.header("Content-Length", "58");*/ conn.header( "Origin" , "http://www.square-bear.co.uk" ); conn.header( "Pragma" , "no-cache" ); conn.header( "Connection" , "Keep-Alive" ); //必须 填写 表单提交 conn.header( "Content-Type" , "application/x-www-form-urlencoded" ); conn.header( "Host" , "fiddle.pandorabots.com" ); conn.header( "Referer" , "http://www.square-bear.co.uk/mitsuku/mitsy_retro.swf" ); conn.header( "User-Agent" , "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36" ); try { Response response = (Response) conn.ignoreContentType( true ).method(Method.POST).data(map).execute(); //response. String json=response.body(); System.out.println(json); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } / //方案2： 通过 开发抓包工具可知 表单的提交方式 应该为http post 此处为 get 方法 不合适 /** * 请求英文对话的网页，抓取结果 * @param url * @return */ private static String processLogic (String url){ String result = "" ; try { Document document = Jsoup.connect(url).ignoreContentType( true ).ignoreHttpErrors( true ) .followRedirects( true ).timeout( 5000 ).userAgent( "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0;)" ).get(); Elements elements = document.select( "result that" ); result = elements.text(); System.out.println(result); } catch (Exception e) { e.printStackTrace(); result = "ok" ; } return result; } public static void main (String[] args) { for ( int i = 0 ; i < 100 ; i++) { // Thread thread=new Thread(); //thread.start(); postMitSuKu(); } /* 通过 开发抓包工具可知 表单的提交方式 应该为http post 此处为 get 方法 不合适 String url = "http://fiddle.pandorabots.com/pandora/talk-xml?input=%s&botid=9fa364f2fe345a10&custid=bbbb30debe1bc7f7"; processLogic(url); */ }}