Jsoup支持Xpath

于 2020-04-19 11:16:40 发布
阅读量848 收藏
点赞数
分类专栏： Java基础 文章标签： xpath
原文链接：https://blog.csdn.net/cysdxy/article/details/52703069
版权

path是专业的xml结构化文档的查询语言，语法功能强大，本文不涉及xpath语法教程。

jsoup 是一款Java 的HTML解析器，可直接解析某个URL地址、HTML文本内容。它提供了一套非常省力的API，可通过DOM，CSS以及类似于jQuery的操作方法来取出和操作数据，但是选取某个元素时还是没有xpath那么简单直接，而且xpath带了很多选择库。

然而遗憾的时，jsoup并不支持xpath，于是博主就写了一个让jsoup支持的xpath的工具类，希望能帮助到有需要的朋友！

工具类
package com.ry.mytools.util;

import com.sun.org.apache.xerces.internal.dom.ElementImpl;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Attribute;
import org.jsoup.select.Elements;
import org.jsoup.select.NodeTraversor;
import org.jsoup.select.NodeVisitor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.*;

import javax.xml.namespace.QName;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

/**
 *  * Jsoup的xpath解析工具类
 *  * 
 *  * @author liuhh
 *  *
 *  
 */
@SuppressWarnings("restriction")
public class JsoupParserUtil {
    protected final static DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
    private final static Logger log = LoggerFactory.getLogger(JsoupParserUtil.class);
    private final static XPath xPath = XPathFactory.newInstance().newXPath();
    protected static TransformerFactory tf = TransformerFactory.newInstance();
    private static final Lock LOCK = new ReentrantLock();

    /**
     * 得到该节点的子节点个数
     */
    public static int getEleChildNum(final org.jsoup.nodes.Element ele, final String xpath) {
        try {
            Object res = parse(ele, xpath, XPathConstants.NODESET);
            if (null != res && res instanceof NodeList) {
                NodeList nodeList = (NodeList) res;
                return nodeList == null ? 0 : nodeList.getLength();

            }
        } catch (Exception e) {
            log.error("根据xpath:{}，获取子节点个数出现错误,错误原因：" + e.getMessage(), xpath);

        }
        return 0;
    }

    /**
     * 判断文档中是否存在xpath节点
     */
    public static boolean exists(final org.jsoup.nodes.Element ele, final String xpath) {
        try {
            Object res = parse(ele, xpath, XPathConstants.BOOLEAN);
            if (null != res && res instanceof Boolean) {
                return (boolean) res;
            }
            return false;
        } catch (Exception e) {
            log.error("检查xpath:{}，是否存在时出现错误,！" + e.getMessage(), xpath);
        }
        return false;
    }

    /**
     * 根据xpath得到w3c的Element对象
     */
    public static ElementImpl getW3cElementImpl(final org.jsoup.nodes.Element ele, final String xpath) {
        try {
            Object res = parse(ele, xpath, XPathConstants.NODE);
            if (null != res && res instanceof ElementImpl) {
                return (ElementImpl) res;
            }
            return null;
        } catch (Exception e) {
            log.error("根据xpath：{}，得到w3c的Element对象出现错误，原因：" + e.getMessage(), xpath);
        }
        return null;
    }

    /**
     * 根据xpath得到jsoup的Element对象
     */
    public static org.jsoup.nodes.Element getJsoupElement(final org.jsoup.nodes.Element ele, final String xpath) {
        try {
            Object res = parse(ele, xpath, XPathConstants.NODE);
            if (null != res && res instanceof ElementImpl) {
                ElementImpl elementImpl = (ElementImpl) res;
                return getJsoupEle(elementImpl);
            }
            return null;
        } catch (Exception e) {
            log.error("根据xpath：{}，得到jsoup的Element对象出现错误，原因：" + e.getMessage(), xpath);
        }
        return null;
    }

    /**
     * 根据xpath得到jsoup的Elements对象
     */
    public static Elements getJsoupElements(final org.jsoup.nodes.Element ele, final String xpath) {
        try {
            NodeList nodeList = getNodeList(ele, xpath);
            if (null != nodeList && nodeList.getLength() > 0) {
                int len = nodeList.getLength();
                Elements elements = new Elements();
                for (int i = 0; i < len; i++) {
                    Node node = nodeList.item(i);
                    if (null != node && node instanceof ElementImpl) {
                        org.jsoup.nodes.Element
                                element = getJsoupEle(((ElementImpl) node));
                        elements.add(element);
                    }
                }
                return elements;
            }

        } catch (Exception e) {
            log.error("根据xpath：{}，得到jsoup的Element对象出现错误，原因：" + e.getMessage(), xpath);
        }
        return null;
    }

    /**
     * 从Jsoup的Element中解析出W3C的NodeList
     */
    public static NodeList getNodeList(final org.jsoup.nodes.Element ele, final String xpath) {
        try {
            Object res = parse(ele, xpath, XPathConstants.NODESET);
            if (null != res && res instanceof NodeList) {
                return (NodeList) res;
            }
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return null;
    }

    /**
     * 得到节点的某一个属性
     */
    public static String getXpathString(final org.jsoup.nodes.Element ele, final String xpath) {
        try {
            int textNum = getEleChildNum(ele, xpath);
            if (1 == textNum) {
                Object res = parse(ele, xpath, XPathConstants.STRING);
                if (null != res) {
                    return res.toString();
                }
            } else {
                List<String> res = getXpathListString(ele, xpath);
                if (res != null && res.size() > 0) {
                    StringBuilder stringBuilder = new StringBuilder();
                    for (Iterator<String> iterator = res.iterator(); iterator.hasNext(); ) {
                        String text = iterator.next();
                        if (null != text) {
                            stringBuilder.append(text.replace("\r\n", "."));
                        }
                    }
                    return stringBuilder.toString();
                }
            }
            return null;
        } catch (Exception e) {
            e.printStackTrace();
            log.error("根据xpath:{}查询字符串时出现错误:" + e.getMessage(), xpath);
        }
        return null;
    }

    /**
     * 查询字符串列表
     */
    public static List<String> getXpathListString(final org.jsoup.nodes.Element ele, final String xpath) {
        try {
            Object res = parse(ele, xpath, XPathConstants.NODESET);
            if (null != res && res instanceof NodeList) {
                NodeList nodeList = (NodeList) res;
                int length = nodeList.getLength();
                if (length <= 0) {
                    return null;
                }
                List<String> list = new ArrayList<>();
                for (int i = 0; i < length; i++) {
                    Node node = nodeList.item(i);
                    list.add(null == node ? null : node.getNodeValue());
                }
                return list;
            }
            return null;
        } catch (Exception e) {
            log.error("根据xpath:{}查询字符串列表时出现错误:" + e.getMessage(), xpath);
        }
        return null;
    }

    /**
     * 获取xpath解析结果
     */
    public static Object parse(final org.jsoup.nodes.Element doc, final String xPathStr, final QName qName) {
        Node node = fromJsoup(doc);
        return parse(node, xPathStr, qName);
    }

    public static Object parse(final Node doc, final String xPathStr, final QName qName) {
        try {
            if (doc == null) {
                log.warn("解析文档为null！");
                return null;
            }
            if (StringUtils.isBlank(xPathStr)) {
                log.warn("解析的Xpath路径为空！");
                return null;
            }
            if (null == qName) {
                log.warn("解析类型为null！");
                return null;
            }
            try {
                LOCK.lock();
                Object res = xPath.evaluate(xPathStr, doc, qName);
                return res;
            } finally {
                // TODO: handle finally clause
                LOCK.unlock();
            }
        } catch (Exception e) {
            log.warn("解析Xpath：{}，出现错误,解析类型：{}，错误原因：{}！", xPathStr, qName, e.getMessage());
        }
        return null;
    }

    /**
     * 根据ElementImpl得到Jsoup的Element
     */
    public static org.jsoup.nodes.Element getJsoupEle(final ElementImpl elementImpl) {
        try {
            String value = getW3cDocString(elementImpl);
            org.jsoup.nodes.Document document = Jsoup.parse(value);
            return document.body().child(0);

        } catch (Exception e) {
            // TODO: handle exception
            log.error("根据ElementImpl得到Jsoup的Element出现错误，错误原因：" + e.getMessage());
            return null;

        }
    }

    /**
     * 将w3c的Document转为jsoup的Document
     */
    public static org.jsoup.nodes.Document fromW3C(final Document doc) throws Exception {
        String string = getW3cDocString(doc);
        org.jsoup.nodes.Document res = Jsoup.parse(string);
        return res;
    }

    /**
     * 将jsoup的Document转为w3c的Document
     */
    public static Node fromJsoup(final org.jsoup.nodes.Element in) {
        DocumentBuilder builder;
        try {
            if (null == in) {
                return null;
            }
            builder = factory.newDocumentBuilder();
            Document out = builder.newDocument();
            if (in instanceof org.jsoup.nodes.Document) {
                List<org.jsoup.nodes.Node> childs = in.childNodes();
                if (childs != null && childs.size() > 0) {
                    org.jsoup.nodes.Element rootEl = in.child(0);
                    NodeTraversor
                            traversor = new NodeTraversor(new W3CBuilder(out));
                    traversor.traverse(rootEl);
                    return out;
                } else {
                    // out.setNodeValue(in.);
                    return out;
                }
            } else if (in instanceof org.jsoup.nodes.Element) {
                NodeTraversor
                        traversor = new NodeTraversor(new W3CBuilder(out));
                traversor.traverse(in);
                return out;
            }
        } catch (ParserConfigurationException e) {
            return null;
        }
        return null;
    }

    /**
     * 将W3c的doc转为字符串
     */
    public static String getW3cDocString(final Node doc) throws Exception {
        try (StringWriter writer = new StringWriter()) {
            DOMSource domSource = new DOMSource(doc);
            StreamResult result = new StreamResult(writer);
            LOCK.lock();
            try {
                Transformer transformer = tf.newTransformer();
                transformer.transform(domSource, result);
                return writer.toString();

            } finally {
                LOCK.unlock();
            }
        } catch (TransformerException e) {
            throw new IllegalStateException(e);
        }
    }

    /**
     * 将Jsoup的node属性拷贝到w3c的Element中
     */
    public static void copyAttributes(final org.jsoup.nodes.Node source, final Element el) {
        for (Attribute attribute : source.attributes()) {
            el.setAttribute(attribute.getKey(), attribute.getValue());
        }
    }
}

class W3CBuilder implements NodeVisitor {
    private final Document doc;
    private Element dest;

    public W3CBuilder(Document doc) {
        this.doc = doc;
    }

    @Override
    public void head(final org.jsoup.nodes.Node source, int depth) {
        if (source instanceof org.jsoup.nodes.Element) {
            org.jsoup.nodes.Element
                    sourceEl = (org.jsoup.nodes.Element) source;
            Element el = doc.createElement(sourceEl.tagName());
            JsoupParserUtil.copyAttributes(sourceEl, el);
            if (dest == null) {
                doc.appendChild(el);
            } else {
                dest.appendChild(el);
            }
            dest = el;

        } else if (source instanceof org.jsoup.nodes.TextNode) {
            org.jsoup.nodes.TextNode
                    sourceText = (org.jsoup.nodes.TextNode) source;
            Text text = doc.createTextNode(sourceText.getWholeText());
            dest.appendChild(text);

        } else if (source instanceof org.jsoup.nodes.Comment) {
            org.jsoup.nodes.Comment
                    sourceComment = (org.jsoup.nodes.Comment) source;
            Comment comment = doc.createComment(sourceComment.getData());
            dest.appendChild(comment);

        } else if (source instanceof org.jsoup.nodes.DataNode) {
            org.jsoup.nodes.DataNode
                    sourceData = (org.jsoup.nodes.DataNode) source;
            Text node = doc.createTextNode(sourceData.getWholeData());
            dest.appendChild(node);

        } else {

        }
    }

    @Override
    public void tail(final org.jsoup.nodes.Node source, int depth) {
        if (source instanceof org.jsoup.nodes.Element && dest.getParentNode() instanceof Element) {
            dest = (Element) dest.getParentNode();
        }
    }
}
测试
import java.io.IOException;
import java.net.URL;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
 
public class JsoupParserUtilsTest {
	
	public static void main(String[] args) throws Exception, IOException {
		String url = "http://mil.news.sina.com.cn/china/2016-09-29/doc-ifxwmamy9955666.shtml";
		Document doc = Jsoup.parse(new URL(url), 10000);
		String titleXpath = "//*[@id='main_title']/text()";
		String timeXpath = "//*[@id='page-tools']/span/span[position() = 1]";
		System.out.println(JsoupParserUtils.exists(doc, "/html/body/div[position>1000000]"));
		System.out.println(JsoupParserUtils.getXpathString(doc, titleXpath));
		Element element = JsoupParserUtils.getJsoupElement(doc, timeXpath);
		System.out.println(element.text());
		System.out.println(element.attr("class"));
	}
}

————————————————
让你的Jsoup支持Xpath

God__is__a__girl
关注
  • 0
    点赞
  • 踩
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
专栏目录
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值