xpath java html_Java 使用HtmlCleaner、Saxon和XPath(XPathEvaluator)进行html查找解析的方法...

1、安装引用HtmlCleaner和Saxon

Maven中Pom.xml中添加依赖：

net.sourceforge.htmlcleaner

htmlcleaner

net.sf.saxon

Saxon-HE

2、使用HtmlCleaner、Saxon和XPath(XPathEvaluator)示例代码

package us.codecraft.webmagic.selector;

import net.sf.saxon.lib.NamespaceConstant;

import net.sf.saxon.xpath.XPathEvaluator;

import org.htmlcleaner.CleanerProperties;

import org..DomSerializer;

import org.htmlcleaner.HtmlCleaner;

import org.htmlcleaner.TagNode;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import org.w3c.dom.Document;

import org.w3c.dom.Node;

import org.w3c.dom.NodeList;

import javax.xml.namespace.NamespaceContext;

import javax.xml.transform.OutputKeys;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.dom.DOMSource;

import javax.xml.transform.stream.StreamResult;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpression;

import javax.xml.xpath.XPathExpressionException;

import java.io.StringWriter;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.List;

import java.util.Map;

import java.util.concurrent.ConcurrentHashMap;

/**

* 支持xpath2.0的选择器。包装了HtmlCleaner和Saxon HE。

*

*/

public class Xpath2Selector implements Selector {

private String xpathStr;

private XPathExpression xPathExpression;

private Logger logger = LoggerFactory.getLogger(getClass());

//指定xpath表达式

public Xpath2Selector(String xpathStr) {

this.xpathStr = xpathStr;

try {

init();

} catch (XPathExpressionException e) {

throw new IllegalArgumentException("XPath error!", e);

}

}

enum XPath2NamespaceContext implements NamespaceContext {

INSTANCE;

private final Map prefix2NamespaceMap = new ConcurrentHashMap();

private final Map> namespace2PrefixMap = new ConcurrentHashMap>();

private void put(String prefix, String namespaceURI) {

prefix2NamespaceMap.put(prefix, namespaceURI);

List prefixes = namespace2PrefixMap.get(namespaceURI);

if (prefixes == null) {

prefixes = new ArrayList();

namespace2PrefixMap.put(namespaceURI, prefixes);

}

prefixes.add(prefix);

}

private XPath2NamespaceContext() {

put("fn", NamespaceConstant.FN);

put("xslt", NamespaceConstant.XSLT);

}

@Override

public String getNamespaceURI(String prefix) {

return prefix2NamespaceMap.get(prefix);

}

@Override

public String getPrefix(String namespaceURI) {

List prefixes = namespace2PrefixMap.get(namespaceURI);

if (prefixes == null || prefixes.size() < 1) {

return null;

}

return prefixes.get(0);

}

@Override

public Iterator getPrefixes(String namespaceURI) {

List prefixes = namespace2PrefixMap.get(namespaceURI);

if (prefixes == null || prefixes.size() < 1) {

return null;

}

return prefixes.iterator();

}

}

private void init() throws XPathExpressionException {

XPathEvaluator xPathEvaluator = new XPathEvaluator();

xPathEvaluator.setNamespaceContext(XPath2NamespaceContext.INSTANCE);

xPathExpression = xPathEvaluator.compile(xpathStr);

}

@Override

public String select(String text) {

try {

HtmlCleaner htmlCleaner = new HtmlCleaner();

TagNode tagNode = htmlCleaner.clean(text);

Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);

Object result;

try {

result = xPathExpression.evaluate(document, XPathConstants.NODESET);

} catch (XPathExpressionException e) {

result = xPathExpression.evaluate(document, XPathConstants.STRING);

}

if (result instanceof NodeList) {

NodeList nodeList = (NodeList) result;

if (nodeList.getLength() == 0) {

return null;

}

Node item = nodeList.item(0);

if (item.getNodeType() == Node.ATTRIBUTE_NODE || item.getNodeType() == Node.TEXT_NODE) {

return item.getTextContent();

} else {

StreamResult xmlOutput = new StreamResult(new StringWriter());

Transformer transformer = TransformerFactory.newInstance().newTransformer();

transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");

transformer.transform(new DOMSource(item), xmlOutput);

return xmlOutput.getWriter().toString();

}

}

return result.toString();

} catch (Exception e) {

logger.error("select text error! " + xpathStr, e);

}

return null;

}

@Override

public List selectList(String text) {

List results = new ArrayList();

try {

HtmlCleaner htmlCleaner = new HtmlCleaner();

TagNode tagNode = htmlCleaner.clean(text);

Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);

Object result;

try {

result = xPathExpression.evaluate(document, XPathConstants.NODESET);

} catch (XPathExpressionException e) {

result = xPathExpression.evaluate(document, XPathConstants.STRING);

}

if (result instanceof NodeList) {

NodeList nodeList = (NodeList) result;

Transformer transformer = TransformerFactory.newInstance().newTransformer();

StreamResult xmlOutput = new StreamResult();

transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");

for (int i = 0; i < nodeList.getLength(); i++) {

Node item = nodeList.item(i);

if (item.getNodeType() == Node.ATTRIBUTE_NODE || item.getNodeType() == Node.TEXT_NODE) {

results.add(item.getTextContent());

} else {

xmlOutput.setWriter(new StringWriter());

transformer.transform(new DOMSource(item), xmlOutput);

results.add(xmlOutput.getWriter().toString());

}

}

} else {

results.add(result.toString());

}

} catch (Exception e) {

logger.error("select text error! " + xpathStr, e);

}

return results;

}

}

使用方法：Xpath2Selector xpath2Selector=new Xpath2Selector("/html/body");

System.out.printf(xpath2Selector.select("

测试数据

"));

相关文档：