html数据抓取,httpclient 抓取html数据

package com.hoperun.webos.servlet;

import java.io.IOException;

import java.io.InputStream;

import java.util.ArrayList;

import java.util.List;

import javax.servlet.ServletException;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import org.apache.http.HttpEntity;

import org.apache.http.HttpResponse;

import org.apache.http.HttpStatus;

import org.apache.http.client.ClientProtocolException;

import org.apache.http.client.HttpClient;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.DefaultHttpClient;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import com.hoperun.webos.pojo.Entity;

import com.thoughtworks.xstream.XStream;

public class WidgetServlet extends HttpServlet {

private static final long serialVersionUID = 1L;

private List categoryArray = new ArrayList();

public WidgetServlet() {

super();

// 待办事宜

categoryArray.add("dbsy");

// 待阅事宜

categoryArray.add("dysy");

// 邮件

categoryArray.add("yj");

}

@Override

protected void service(HttpServletRequest request,

HttpServletResponse response) throws ServletException, IOException {

request.setCharacterEncoding("utf-8");

// 请求参数在数组中的下标

int index = 0;

// 获取要提取的信息分类

String param = request.getParameter("category");

index = categoryArray.indexOf(param);

// 参数值大于等于零

if (index >= 0) {

Document doc = getDocument();

// 解析class=txt_12_black的节点，

// 总共为八个结果，我们只需要前六个，根据index的值来选择要解析的element，计算公式为2*i+1

Elements elements = doc.select(".txt_12_black");

Element element = elements.get(2 * index + 1);

// 详细解析

List result = detailParse(element);

// 将结果写到response中

response.setCharacterEncoding("utf-8");

response.setContentType("text/html;charset=utf-8");

String finalResult = toXML(result);

// 将result转化为xml格式

response.getWriter().write(finalResult);

}

}

/** 获取url所返回的页面数据 */

private Document getDocument() {

Document doc = null;

HttpClient httpClient = new DefaultHttpClient();

HttpGet httpGet = new HttpGet(

"http://10.20.107.102:8080/res/FrmRight_01.htm");

try {

HttpResponse httpResponse = httpClient.execute(httpGet);

// 返回成功

if (HttpStatus.SC_OK == httpResponse.getStatusLine()

.getStatusCode()) {

// 得到输入流

HttpEntity httpEntity = httpResponse.getEntity();

InputStream is = httpEntity.getContent();

// 解析为html文档

doc = Jsoup.parse(is, "gb2312", "http://10.20.107.102:8080");

}

} catch (ClientProtocolException e) {

e.printStackTrace();

} catch (IOException e) {

e.printStackTrace();

}

return doc;

}

/** 对element进行详细解析 */

private List detailParse(Element element) {

// 返回的结果实体集

List array = new ArrayList();

Elements elem = element.children();

// 返回记录的分类

String category = null;

for (Element temp : elem) {

// elem直接子元素的标签为tbody

if ("tbody".equals(temp.tagName())) {

Elements tbChild = temp.children();

Element categoryElement = null;

// 当他的直接子元素有两个时，第二个为该记录集合的分类信息，否则记录的分类信息位于第四个直接记录中

if (tbChild.size() == 2) {

categoryElement = tbChild.get(1);

category = categoryElement.text();

}

} else if ("a".equals(temp.tagName())) { // elem直接子元素的标签为a

Entity entity = new Entity();

// 记录项前面是否有选择框

int flag = 0;

Elements as = temp.child(0).child(0).child(0).children();

// 当第一个直接子元素的文本值为空时，则第一个直接子元素为选择框

if (null == as.get(0).text() || "".equals(as.get(0).text())) {

flag = 1;

}

entity.setData1(as.get(0 + flag).text());

entity.setDate2(as.get(1 + flag).text());

entity.setData3(as.get(2 + flag).text());

// 处理待阅事宜前面的选择框

if (as.size() == 4 && flag == 0) {

// 该记录的分类信息位于第四个直接子元素

entity.setData4(as.get(3).text());

} else {

// 该记录的分类信息是分类集合中的一条，所以分类信息位于tbody中

entity.setData4(category);

}

// 格式化地址

String href = temp.attr("href");

entity.setData5("http://10.20.107.102:8080" + formatURL(href));

array.add(entity);

} else if ("tr".equals(temp.tagName())) { // elem直接子元素的标签为tr

// 标签里面包含该记录的分类信息

category = temp.text();

}

}

return array;

}

/** 将result转化为xml格式 */

private String toXML(List array) {

XStream xstream = new XStream();

xstream.alias("item", Entity.class);

if (null == array) {

array = new ArrayList();

}

return xstream.toXML(array);

}

/** 格式化地址 */

private String formatURL(String href) {

int startIndex = href.indexOf("\'");

int endIndex = href.lastIndexOf("\'");

return href.substring(startIndex + 1, endIndex);

}

}