有两个问题,记录一下 :
1、xpath如下格式(通过firebug下面的xpath插件获得的路径):
1 | html/body/div[2]/div[1]/div[3]/ul/li/div/div/span/span[2] |
此时htmlcleaner调用xpath查找元素的方法找不到该元素(确定及肯定该界面有这元素),将路径修改为:
1 | /body/div[2]/div[1]/div[3]/ul/li/div/div/span/span[2] |
就可以爬取到了。
2、爬取元素的代码:(记录一下啦)
01 | Document doc = getDocument(url); |
02 | if (doc!= null && doc.hasText()){ |
03 | // System.out.println(doc.html()); |
04 | String document = formatContext(doc.html()); |
05 | System.out.println(document); |
06 | |
07 | TagNode node = cleaner.clean(document); |
08 |
09 | Object[] ns = null ; |
10 | try { |
11 | ns = node.evaluateXPath(xpath); |
12 | } catch (XPatherException e) { |
13 | e.printStackTrace(); |
14 | } |
15 | if (ns.length > 0 ) { |
16 | for (Object on : ns) { |
17 | TagNode n = (TagNode) on; |
18 | //下面的两个方法都可以取到标签里面的内容 |
19 | String text = cleaner.getInnerHtml(n); |
20 | System.out.println(text); |
21 | System.out.println(url + n.getText().toString()); |
22 |
23 | } |
24 | } else { |
25 | return null ; // 规则不对,返回空 |
26 | } |