htmlparser网页解析实例

 今天又看了一下htmlparser对网页的解析，发现越看越不会，例如吧，有以下两个网页：

<font color="#808080">2005-2-25 11:06:27</font>和

<h2 class="ContentAuthor">作者:蜘蛛日期:2007-09-20</h2>

看上去吧格式相同啊！为什么总是不能用同一种方法把它们解析呢？？？？？？

看一下代码吧；

1. import java.io.BufferedWriter;
2. import java.io.File;
3. import java.io.FileWriter;
4. import java.io.IOException;
6. import org.htmlparser.Node;
7. import org.htmlparser.NodeFilter;
8. import org.htmlparser.Parser;
9. import org.htmlparser.filters.AndFilter;
10. import org.htmlparser.filters.HasAttributeFilter;
11. import org.htmlparser.filters.HasChildFilter;
12. import org.htmlparser.filters.OrFilter;
13. import org.htmlparser.filters.TagNameFilter;
14. import org.htmlparser.tags.HeadingTag;
15. import org.htmlparser.util.NodeList;
16. import org.htmlparser.visitors.TextExtractingVisitor;
18. import com.extractor.Extractor;
20. public class ExtractorHang extends Extractor{
21.     public void extract()
22.     {
23.         BufferedWriter bw=null;
24.         String indextime="";
25.         String title="";
26.         StringBuffer body=new StringBuffer();
27.         NodeFilter time_filter=new OrFilter(new AndFilter(new TagNameFilter("h2"),new HasAttributeFilter("class","ContentAuthor")),
28.                 new AndFilter(new TagNameFilter("font"),new HasAttributeFilter("color","#808080")));
29.         
30.         NodeFilter title_filter1=new OrFilter(new AndFilter(new TagNameFilter("td"),new HasChildFilter(new TagNameFilter("b"))),
31.                 new AndFilter(new TagNameFilter("h1"),new HasChildFilter(new TagNameFilter("strong"))));//new AndFilter(new HasAttributeFilter("color","#000080")
32.         
33.         NodeFilter  body_filter=new OrFilter(new AndFilter(new TagNameFilter("div"),new HasAttributeFilter("id","logPanel")),
34.                 new AndFilter(new TagNameFilter("td"),new HasChildFilter(new TagNameFilter("p"))));
35.         
36.         try
37.         {
38.             NodeList title_nodes=this.getParser().parse(title_filter1);
39.             //遍历所有节点
40.             
41.                /*
42.                 * 取得符合过滤条件的第一个结果
43.                 */
44.                 Node node=title_nodes.elementAt(0);
45.                 /*
46.                  * 把符合条件的tag之间的所有孩子节点进行列表。
47.                  */
48.                 NodeList node2=node.getChildren(); 
49.                 
50.                 //title=node2.elementAt(0).toHtml();
51.                 /*
52.                  * 博客解析，取值
53.                  */
54.                 title=node2.elementAt(1).toHtml();   
55.                // title=node2.elementAt(2).toHtml(); 
56.                 /*
57.                  * 主页解析
58.                  */
59.                 if(title.indexOf("<")!=-1)
60.                 title=node2.elementAt(3).toHtml();   
62.             /*
63.              * 写入文件。
64.              */
65.             bw=new BufferedWriter(new FileWriter(new File(this.getOutputPath()+title+".txt")));
66.             
67.             int end1=getInputFilePath().lastIndexOf("i");
68.             int end2=getInputFilePath().lastIndexOf(".");
69.             String url_seg1=getInputFilePath().substring(3,end1);
70.             String url_seg2=getInputFilePath().substring(end1, end2);
71.             String url_seg=url_seg1+".asp?"+url_seg2;
72.             url_seg=url_seg.replaceAll("","/");
73.             String url="http://"+url_seg;
74.             
75.             bw.write(url+NEWLINE);
76.             bw.write(title+NEWLINE);
77.             
78.         
79.         }
80.         catch(Exception e)
81.         {
82.             e.printStackTrace();
83.         }
84.         
85.         this.getParser().reset();
86.         try
87.         {
88.             NodeList time_nodes=this.getParser().parse(time_filter);
89.             for(int i=0;i<time_nodes.size();i++)
90.             {
91.                /*
92.                 * 不太好，有待改进。
93.                 */
94.                 Node time_node=time_nodes.elementAt(i);
95.                 indextime=time_node.getNextSibling().toHtml();
96.                 
97.                 /*
98.                 Node node=time_nodes.elementAt(i); 
99.                 NodeList node1=node.getChildren(); 
100.                 indextime=node1.elementAt(0).toHtml();
101.                 */
102.                 /*
103.                  * 用Node类，其对象不能用getChildren。
104.                  * 而用tag类，则可以。
105.                  */
106.                 if(indextime.indexOf("日期")!=-1)
107.                 {
108.                     indextime=indextime.substring(indextime.lastIndexOf(":")+1);
109.                 }
110.             }
111.             
112.             bw.write(indextime+NEWLINE);
113.         }
114.         catch(Exception e)
115.         {
116.             e.printStackTrace();
117.         }
118.         
119.         this.getParser().reset();
120.         try
121.         {
122.             NodeList body_nodes=this.getParser().parse(body_filter);
123.             for(int i=0;i<body_nodes.size();i++)
124.             {
125.                 Node node=body_nodes.elementAt(i);
126.                 
127.                 Parser body_parser=new Parser(node.toHtml());
128.                 TextExtractingVisitor visitor=new TextExtractingVisitor();
129.                 body_parser.visitAllNodesWith(visitor);
130.                 body.append(visitor.getExtractedText());
131.             }
132.             String text=body.toString();
133.             String newtext="";
134.             /*while(text.indexOf("<")!=-1)
135.             {
136.                 text=text.substring(0, body.indexOf("<"));
137.                 text+=text.substring(75);
138.             }
139.             */
140.             /*
141.              * 循环，有点耗时了。
142.              * 主要是为了除去文本中
143.              * <?xml:namespace prefix = o ns = "urn:schemas-microsoft-com:office:office" />
144.              * 字段。顺便把一些没必要的空格去掉。
145.              */
146.             for(int i=0;i<text.length();i++)
147.             {
148.                 char ch=text.charAt(i);
149.                 if(ch=='<')
150.                 {
151.                     i++;
152.                     while(text.charAt(i)!='>')
153.                         i++;
154.                 }
155.                 
156.                 if(ch!='<'&&ch!=' ')
157.                 newtext+=ch;
158.             }
159.             
160.             bw.write(newtext+NEWLINE);
161.         
162.         }
163.         catch(Exception e)
164.         {
165.             e.printStackTrace();
166.         }
167.         
168.         try
169.         {
170.             if(bw!=null)
171.                 bw.close();
172.         }catch(IOException e)
173.         {
174.             e.printStackTrace();
175.         }
176.     }
177. }

上述代码的time_filter部分还有待研究……