直接上代码
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
public class JsoupHelper {
public static Object fecthNode(String url, String xpath) throws Exception {
String html = null;
try {
Connection connect = Jsoup.connect(url);
html = connect.get().body().html();
} catch (IOException e) {
e.printStackTrace();
return null;
}
// System.err.println(html);
HtmlCleaner hc = new HtmlCleaner();
TagNode tn = hc.clean(html);
Document dom = new DomSerializer(new CleanerProperties()).createDOM(tn);
XPath xPath = XPathFactory.newInstance().newXPath();
Object result = xPath.evaluate(xpath, dom, XPathConstants.NODESET);
return result;
}
public static List<GoodsTemp> fecth2Object(String url, String xpath) throws Exception {
List<GoodsTemp> list = new ArrayList<GoodsTemp>();
Object result = fecthNode(url, xpath);
if (result instanceof NodeList) {
NodeList nodeList = (NodeList) result;
for (int i = 0; i < nodeList.getLength(); i++) {
GoodsTemp goods = new GoodsTemp();
Node node = nodeList.item(i);
if(node == null){
continue;
}
String detail = node.getAttributes().getNamedItem("href").getTextContent();
/*nodeMap.put(node.getTextContent(), node.getAttributes().getNamedItem("href")!=null ?
detail : "");*/
goods.setName(node.getTextContent());
goods.setDetailUrl(detail);
goods.setId(detail .replace("/products/", ""));
list.add(goods);
}
}
return list;
}
public static Map<String, String> fecthByMap(String url, String xpath) throws Exception {
Map<String, String> nodeMap = new LinkedHashMap<>();
Object result = fecthNode(url, xpath);
if (result instanceof NodeList) {
NodeList nodeList = (NodeList) result;
for (int i = 0; i < nodeList.getLength(); i++) {
Node node = nodeList.item(i);
if(node == null){
continue;
}
nodeMap.put(node.getTextContent(), node.getAttributes().getNamedItem("href")!=null ?
node.getAttributes().getNamedItem("href").getTextContent() : "");
//System.out.println(node.getTextContent() + " : " + node.getAttributes().getNamedItem("href"));
}
}
return nodeMap;
}
/**
*获取xpath下的某个属性值
**/
public static List<String> fecthAttr(String url, String xpath, String attr) throws Exception {
List<String> list = new ArrayList<>();
Object result = fecthNode(url, xpath);
if (result instanceof NodeList) {
NodeList nodeList = (NodeList) result;
for (int i = 0; i < nodeList.getLength(); i++) {
Node node = nodeList.item(i);
if(node == null){
continue;
}
list.add(node.getAttributes().getNamedItem(attr).getTextContent());
//System.out.println(node.getTextContent() + " : " + node.getAttributes().getNamedItem("href"));
}
}
return list;
}
public static void main(String[] args) throws Exception {
/*String xpath = "//div[@class='gp-list-view-search']/ul/li//div[@class='info']/div[@class='title']/a[1]";
Map map = fecthByMap ("", xpath);
//Map map = fecthByMap("","//ul[@class='note-list']/li//a[@class='title']");
Iterator<Map.Entry<String, String>> it=map.entrySet().iterator();
while(it.hasNext()){
Map.Entry<String, String> entry=it.next();
System.out.println("键key :"+entry.getKey()+" value :"+entry.getValue());
}
System.out.println("====================================");*/
String xpath = "//div[@class='gp-list-view-search']/ul/li//div[@class='info']/div[@class='title']/a[1]";
List<GoodsTemp> list = fecth2Object("", xpath);
for (GoodsTemp goodsTemp : list) {
System.err.println(goodsTemp);
}
}
}
maven配置如下:
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>1.4.3</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.10.3</version>
</dependency>
<dependency>
<groupId>javax.xml</groupId>
<artifactId>jaxp-api</artifactId>
<version>1.4.2</version>
</dependency>
<dependency>
<groupId>net.sourceforge.htmlcleaner</groupId>
<artifactId>htmlcleaner</artifactId>
<version>2.9</version>
</dependency>
有时会出现获取不到问题,主要就是xpath有问题,有可能直接在页面看到的html和通过程序返回的并不完全一致,建议根据程序返回的写xpath