packagecn.tdt.crawl.jdbc;importjava.util.regex.Matcher;importjava.util.regex.Pattern;importorg.jsoup.Jsoup;importorg.jsoup.nodes.Document;importorg.jsoup.nodes.Element;importorg.jsoup.select.Elements;public classHtmlExtract {private static double linkTextRadio = 0.25; //链接文字比//过滤不必要的数据
public staticString filterContent(String str) {if (str == "") {return "";
}
str= str.replaceAll("(?is)*?>", "");
str= str.replaceAll("(?is)", "");
str= str.replaceAll("(?is).*?", "");
str= str.replaceAll("(?is).*?", "");//str=str.replaceAll("&.{2,5};|.{2,5};", " ");
returnstr;
}//计算链接数
public static intcalcLinks(Element node) {
Elements links= node.select("a[href]");returnlinks.size();
}//计算内容长度
public static doublecalcWords(Element node) {
String con=node.text();if (con.length() == 0) {return 1 +linkTextRadio;
}else{returncon.length();
}
}//计算标点符号的个数
public static intcalcSign(Element node) {
String[] sign= { ",", ";", ".", "\"", "'", "\\?", "。", ":", ","};int i = 0;for(String ch : sign) {int count = 0;
count= node.text().split(ch).length - 1;
i= +count;
}returni;
}//将所有的空节点全部删除
public staticElement drawCon(Element node) {if (node.tagName() == "a") {//这个就不用进去深入了
returnnode;
}int links; //链接数
double words; //文字长度
doublecellRatio;int signs; //符号出现的情况
Elements nodes=node.children();for(Element cnode : nodes) {if (!cnode.hasText()) {//System.out.println("删除"+cnode);
cnode.remove();
}else{
links=calcLinks(cnode);
words=calcWords(cnode);
cellRatio= links /words;
signs=calcSign(cnode);if (signs < 1) {//删除没有标点符号的节点
cnode.remove();
}else if (cellRatio >linkTextRadio) {
cnode.remove();
}else{
drawCon(cnode);
}
}
}returnnode;
}//提取标题
privateString drawTitle(String str) {//TODO Auto-generated method stub//先取页面的title部分的值
if (str.length() < 1) {return null;
}
String tit= "";int xhpos = -1; //下划线的位置
int zhpos = -1; //中横线的位置
Pattern pt = Pattern.compile("
(.*)",Pattern.CASE_INSENSITIVE);
Matcher mc=pt.matcher(str);if(mc.find()) {
tit= mc.group(1).trim();//下面需要过滤一下,有些标题会加上下划线或者中横线
xhpos = tit.indexOf("_");
zhpos= tit.indexOf("|");if (xhpos > 0) {
tit= tit.substring(0, xhpos);
}if (zhpos > 0) {
tit= tit.substring(0, zhpos);
}
}returntit;
}//提取作者
privateString[] drawAuthor(String str) {if (str.length() < 1) {return null;
}//这种信息一般就是直接用正则就好
String[] author = new String[2];int tPos = 0; //日期所在的位置
Pattern pt =Pattern.compile("作者.+(\\d{4}[-|年]\\d{1,2}[-|月]\\d{1,2}[日]?)",
Pattern.CASE_INSENSITIVE);
Matcher mc=pt.matcher(str);if(mc.find()) {//System.out.println("123");
author[0] = mc.group(1); //存储日期信息
tPos = mc.group().trim().indexOf(author[0]);
author[1] = mc.group().trim().substring(0, tPos);returnauthor;
}return null;
}//核心处理函数
publicString[] extract(String str) {
String title;//标题//String author = "";//作者//String uptime = "";//发布时间
String content; //正文//String[] authors = new String[2];
str=filterContent(str);
Document doc=Jsoup.parse(str);//取body
Element bodynode = doc.select("body").first();
title=drawTitle(str);//authors = drawAuthor(str);//开始遍历节点,进行去噪处理,抽取正文
content =drawCon(bodynode).text();//防止溢出//if (authors.length > 1) {//author = authors[1];//uptime = authors[0];//}//System.out.println(title);//System.out.println(author);//System.out.println(uptime);//System.out.println(content);
String[] arr = new String[2];
arr[0] =title;
arr[1] =content;returnarr;
}public static voidmain(String[] args){
}
}