htmlparser网页解析实例

最新推荐文章于 2021-02-09 05:09:28 发布

caoxu1987728

最新推荐文章于 2021-02-09 05:09:28 发布

阅读量2.9k

点赞数

分类专栏： Search Engine 文章标签： newline filter url office class null

本文链接：https://blog.csdn.net/caoxu1987728/article/details/3016042

版权

Search Engine 专栏收录该内容

28 篇文章 0 订阅

订阅专栏

今天又看了一下htmlparser对网页的解析，发现越看越不会，例如吧，有以下两个网页：

<font color="#808080">2005-2-25 11:06:27</font>和

看上去吧格式相同啊！为什么总是不能用同一种方法把它们解析呢？？？？？？

看一下代码吧；

 
 import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.HasChildFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.HeadingTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.visitors.TextExtractingVisitor;
import com.extractor.Extractor;
public class ExtractorHang extends Extractor{
    public void extract()
    {
        BufferedWriter bw=null;
        String indextime="";
        String title="";
        StringBuffer body=new StringBuffer();
        NodeFilter time_filter=new OrFilter(new AndFilter(new TagNameFilter("h2"),new HasAttributeFilter("class","ContentAuthor")),
                new AndFilter(new TagNameFilter("font"),new HasAttributeFilter("color","#808080")));
        
        NodeFilter title_filter1=new OrFilter(new AndFilter(new TagNameFilter("td"),new HasChildFilter(new TagNameFilter("b"))),
                new AndFilter(new TagNameFilter("h1"),new HasChildFilter(new TagNameFilter("strong"))));//new AndFilter(new HasAttributeFilter("color","#000080") 
        
        NodeFilter  body_filter=new OrFilter(new AndFilter(new TagNameFilter("div"),new HasAttributeFilter("id","logPanel")),
                new AndFilter(new TagNameFilter("td"),new HasChildFilter(new TagNameFilter("p"))));
        
        try
        {
            NodeList title_nodes=this.getParser().parse(title_filter1);
            //遍历所有节点 
            
               /*
                * 取得符合过滤条件的第一个结果
                */
                Node node=title_nodes.elementAt(0);
                /*
                 * 把符合条件的tag之间的所有孩子节点进行列表。
                 */
                NodeList node2=node.getChildren(); 
                
                //title=node2.elementAt(0).toHtml(); 
                /*
                 * 博客解析，取值
                 */
                title=node2.elementAt(1).toHtml();   
               // title=node2.elementAt(2).toHtml();  
                /*
                 * 主页解析
                 */
                if(title.indexOf("<")!=-1)
                title=node2.elementAt(3).toHtml();   
            /*
             * 写入文件。
             */
            bw=new BufferedWriter(new FileWriter(new File(this.getOutputPath()+title+".txt")));
            
            int end1=getInputFilePath().lastIndexOf("i");
            int end2=getInputFilePath().lastIndexOf(".");
            String url_seg1=getInputFilePath().substring(3,end1);
            String url_seg2=getInputFilePath().substring(end1, end2);
            String url_seg=url_seg1+".asp?"+url_seg2;
            url_seg=url_seg.replaceAll("","/");
            String url="http://"+url_seg;
            
            bw.write(url+NEWLINE);
            bw.write(title+NEWLINE);
            
        
        }
        catch(Exception e)
        {
            e.printStackTrace();
        }
        
        this.getParser().reset();
        try
        {
            NodeList time_nodes=this.getParser().parse(time_filter);
            for(int i=0;i<time_nodes.size();i++)
            {
               /*
                * 不太好，有待改进。
                */
                Node time_node=time_nodes.elementAt(i);
                indextime=time_node.getNextSibling().toHtml();
                
                /*
                Node node=time_nodes.elementAt(i); 
                NodeList node1=node.getChildren(); 
                indextime=node1.elementAt(0).toHtml();
                */
                /*
                 * 用Node类，其对象不能用getChildren。
                 * 而用tag类，则可以。
                 */
                if(indextime.indexOf("日期")!=-1)
                {
                    indextime=indextime.substring(indextime.lastIndexOf(":")+1);
                }
            }
            
            bw.write(indextime+NEWLINE);
        }
        catch(Exception e)
        {
            e.printStackTrace();
        }
        
        this.getParser().reset();
        try
        {
            NodeList body_nodes=this.getParser().parse(body_filter);
            for(int i=0;i<body_nodes.size();i++)
            {
                Node node=body_nodes.elementAt(i);
                
                Parser body_parser=new Parser(node.toHtml());
                TextExtractingVisitor visitor=new TextExtractingVisitor();
                body_parser.visitAllNodesWith(visitor);
                body.append(visitor.getExtractedText());
            }
            String text=body.toString();
            String newtext="";
            /*while(text.indexOf("<")!=-1)
            {
                text=text.substring(0, body.indexOf("<"));
                text+=text.substring(75);
            }
            */
            /*
             * 循环，有点耗时了。
             * 主要是为了除去文本中
             * <?xml:namespace prefix = o ns = "urn:schemas-microsoft-com:office:office" />
             * 字段。顺便把一些没必要的空格去掉。
             */
            for(int i=0;i<text.length();i++)
            {
                char ch=text.charAt(i);
                if(ch=='<')
                {
                    i++;
                    while(text.charAt(i)!='>')
                        i++;
                }
                
                if(ch!='<'&&ch!=' ')
                newtext+=ch;
            }
            
            bw.write(newtext+NEWLINE);
        
        }
        catch(Exception e)
        {
            e.printStackTrace();
        }
        
        try
        {
            if(bw!=null)
                bw.close();
        }catch(IOException e)
        {
            e.printStackTrace();
        }
    }
}
 