HtmlparseUtil.java

最新推荐文章于 2022-08-18 18:51:36 发布

gundumw100

最新推荐文章于 2022-08-18 18:51:36 发布

阅读量122

点赞数

分类专栏： j2ee+apache 文章标签： Java J# Blog

本文链接：https://blog.csdn.net/gundumw100/article/details/83664923

版权

j2ee+apache 专栏收录该内容

24 篇文章 0 订阅

订阅专栏

该类并不是一个通用的工具类，需要按自己的要求实现，这里只记录了Htmlparse.jar包的一些用法。仅此而已！
详细看这里：[url]http://gundumw100.iteye.com/blog/704311[/url]


import java.util.*;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.BodyTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

/**
 * httpclient与htmlparse对网页的解析
 * 
 * @author Administrator
 * 
 */
public class HtmlparseUtil {
	WebHttpClient util=new WebHttpClient();
	/**
	 * 获得网页中的超链接，将href和text保存在Map中：map(href,text)
	 * @param url
	 * @param charset
	 * @return
	 */
	public Map<String, String> linkGet(String url, String charset) {
		String content=util.getWebContentByGet(url,charset);
		Map<String, String> linkMap = new HashMap<String, String>();
		try {
			//开始解析
			Parser parser = Parser.createParser(content, charset);
			// 过滤出<a></a>标签
			NodeFilter linkFilter = new NodeClassFilter(LinkTag.class);
			NodeList list = parser.extractAllNodesThatMatch(linkFilter);
			Node node = null;
			for (int i = 0; i < list.size(); i++) {
				node = list.elementAt(i);
				// 获得网页中的链接map(href,text)
				linkMap.put(((LinkTag) node).getLink(), this.processText(((LinkTag) node).getLinkText()));
			}
		} catch (ParserException e) {
			e.printStackTrace();
		} 
		return linkMap;
	}

	/**
	 * 获得网页<body></body>标签中的内容， 保存在body中
	 * @param url
	 * @param charset
	 * @return
	 */
	public String bodyGet(String url, String charset) {
		String content=util.getWebContentByGet(url,charset);
		String body = "";
		try {
			Parser parser = Parser.createParser(content, charset);
			// 过滤<body></body>标签
			NodeFilter bodyFilter = new NodeClassFilter(BodyTag.class);
			NodeList list = parser.extractAllNodesThatMatch(bodyFilter);
			Node node = null;
			for (int i = 0; i < list.size(); i++) {
				node = list.elementAt(i);
				// 获得网页内容 保存在content中
				body = ((BodyTag) node).getBody();
			}
		} catch (ParserException e) {
			e.printStackTrace();
		}
		return body;
	}

	/**
	 * 过滤出class为term的<span>元素,并获得他们的文本
	 * @param url
	 * @param charset
	 * @return
	 */
	public Map<String,String> termGet(String url, String charset) {
		String content=util.getWebContentByGet(url,charset);

		Map<String, String> map = new HashMap<String, String>();
		try {
			//开始解析
			// 过滤出class为term的<span>元素
			Parser parser = Parser.createParser(content, charset);
			AndFilter filter = 
                new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("class","term"));

			Node node = null;
			NodeList nodeList = parser.parse(filter);

			for (int i = 0; i < nodeList.size(); i++) {
				node = nodeList.elementAt(i);
				map.put("term", node.toPlainTextString());
			}
			// 过滤出class为start-time的<span>元素
			Parser parser2 = Parser.createParser(content, charset);
			AndFilter filter2 = 
                new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("class","start-time"));
			NodeList nodeList2 = parser2.parse(filter2);
			for (int i = 0; i < nodeList2.size(); i++) {
				node = nodeList2.elementAt(i);
				map.put("start-time", node.toPlainTextString());
			}
			// 过滤出id为J_SingleEndTimeLabel的<span>元素
			Parser parser3 = Parser.createParser(content, charset);
			AndFilter filter3 = 
                new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("id","J_SingleEndTimeLabel"));
			NodeList nodeList3 = parser3.parse(filter3);
			for (int i = 0; i < nodeList3.size(); i++) {
				node = nodeList3.elementAt(i);
				map.put("end-time", node.toPlainTextString());
			}

			// 过滤出class为box post的<div>元素
			Parser parser4 = Parser.createParser(content, charset);
			AndFilter filter4 = 
                new AndFilter(new TagNameFilter("div"),new HasAttributeFilter("class","box post"));
			NodeList nodeList4 = parser4.parse(filter4);
			for (int i = 0; i < nodeList4.size(); i++) {
				node = nodeList4.elementAt(i);
				String temp=node.toPlainTextString().trim();
				temp=temp.substring(10,20).trim();
				map.put("pre-term", temp);
			}

			// 过滤出class为J_AwardNumber的<span>元素
			Parser parser5 = Parser.createParser(content, charset);
//			AndFilter filter5 = 
//                new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("class","J_AwardNumber"));
			NodeList nodeList5 = parser5.parse(new HasAttributeFilter("class","J_AwardNumber"));
			StringBuffer buffer=new StringBuffer();
			for (int i = 0; i < nodeList5.size(); i++) {
				node = nodeList5.elementAt(i);
				buffer.append(","+node.toPlainTextString());
			}
			buffer.append("|");

			// 过滤出class为blue J_AwardNumber的<span>元素
			Parser parser6 = Parser.createParser(content, charset);
//			AndFilter filter6 = 
//                new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("class","blue J_AwardNumber"));
			NodeList nodeList6 = parser6.parse(new HasAttributeFilter("class","blue J_AwardNumber"));
			for (int i = 0; i < nodeList6.size(); i++) {
				node = nodeList6.elementAt(i);
				buffer.append(node.toPlainTextString()+",");
			}

			map.put("numbers", buffer.toString());
		} catch (ParserException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		return map;
	}

	private String processText(String content){   
		content=content.trim().replaceAll(" ", "");   
//		content=content.replaceAll("<p>", "\n");   
//		content=content.replaceAll("</TD>", "");   
//		content=content.replaceAll("</div>", "");   
//		content=content.replaceAll("</a>", "");   
//		content=content.replaceAll("<a href=.*>", "");   
        return content;   
    }   

	public static void main(String[] str) {

		String url="http://caipiao.taobao.com/lottery/order/lottery_dlt.htm?type=1";
		HtmlparseUtil util=new HtmlparseUtil();
		Map<String,String> map=util.termGet(url, "gb2312");
		System.out.println("term="+map.get("term"));//<span class="term">第<em>10074</em>期</span>
		System.out.println("start-time="+map.get("start-time"));//
		System.out.println("end-time="+map.get("end-time"));//
		System.out.println("pre-term="+map.get("pre-term"));//
		System.out.println("numbers="+map.get("numbers"));//

		/*
		Map<String, String> linkMap = util.linkGet(url, "gb2312");
		for (String s : linkMap.keySet()) {
			System.out.println(s + " = " + linkMap.get(s));
			//如果是个链接，则再获取它的<body>中的内容
//			if (s.startsWith("http")) {
//				util.bodyGet(s, "gb2312");
//			}
		}
		*/

	}

}