抓取网页数据

需要htmlParser.jar。代码如下

public class GetData {

public static void main(String[] args) throws ParserException {

ArrayList<UrlEntity> result = getYellowList("http://www.71hy.com/hy-34-p1.html");

}

        public static boolean getStringsByRegex(String txt, String regex) {//根据匹配规则判断是否 匹配

        Pattern p = Pattern.compile(regex);

        Matcher m = p.matcher(txt);

        if (m.find()) {

            return true;

        }

        return false;

    }

public static ArrayList<UrlEntity> getYellowList(String url){

final ArrayList<UrlEntity> result = new ArrayList<UrlEntity>();

Parser parser = null;

NodeList nodeList = null;

try {

parser = new Parser(url);

parser.setEncoding("gb2312");//设置编码格式

nodeList = parser.parse(

new NodeFilter(){

@Override

public boolean accept(Node node) {

String regex="div class=\"box\"";// 类似 "div id=\"box\"","ulclass=\"box\""等

if(getStringsByRegex(node.getText(),regex)){//如果匹配你要的内容

try {

String html = node.toHtml();

Parser parser2 = Parser.createParser(html, "gb2312");

NodeFilter filter = new TagNameFilter("A");

                                                                        // 在你需要的div 内 过滤出A 标签  返回NodeList

NodeList nodeList3 = parser2.extractAllNodesThatMatch(filter);

for (int i = 0; i<nodeList3.size(); i++) {

LinkTag tag = (LinkTag) nodeList3.elementAt(i);//将node转为相应的 标签

String des = tag.getLinkText();//获取a 标签的内容

if(!des.equals("") && (des.indexOf("[") == -1)){

UrlEntity PageData = new UrlEntity();

PageData.url = tag.getLink();//获取a 标签的href

PageData.title = tag.getAttribute("title");//根据属性获取需要的内容 

PageData.description = des;

PageData.type = "Yellow";

result.add(PageData);

}

}

} catch (ParserException e) {

} 

return true;

}

return false;

}

}

);

}catch (ParserException e) {

}

return result;

}

}

转载于:https://my.oschina.net/u/1387306/blog/215415