HttpClient抓取网页文件方法

最新推荐文章于 2016-11-03 14:13:40 发布

冬天的白雪

最新推荐文章于 2016-11-03 14:13:40 发布

阅读量1.1k

点赞数

文章标签： string exception regex byte url class

本文链接：https://blog.csdn.net/zhldt2008/article/details/6202753

版权

用httpclient-4.0.jar,httpcore-4.0.1.jar抓取页面内容的方法
public class ParsePage {

/**
*
* 根据baseUrl读取页面信息
*
* @throws Exception
*/
public String parseURLContent(String url)
   throws Exception {
  String resultString = "";
  HttpClient client = new DefaultHttpClient();
  HttpUriRequest request = new HttpGet(url);
  HttpResponse response = client.execute(request);
  HttpEntity entity = response.getEntity();
  InputStream in = entity.getContent();
  byte bytes[] = new byte[1024 * 1024];

  while (-1 != in.read(bytes, 0, 1024 * 1024))
  {
   resultString += new String(bytes, "UTF-8");
  }
  return this.parseResult(resultString);

}

// 提取页面信息
private String getResponseMessage(String result) {
  String regex = "<title>.*?</title>";
  Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
  Matcher ma = pa.matcher(result);
  while (ma.find()) {
   return ma.group();
  }
  return "";
}