关闭

C#实现web信息抓取

530人阅读 评论(0) 收藏 举报
        随着Internet的普及,网络信息正以极高的速度增长,在这么多数据中找到自己需要的信息是一件很繁琐的事情,找到需要的信息后如何获取也是件麻烦的事。这就需要Internet信息抓取程序来代替人工的操作。
            所谓Internet信息抓取程序,就是程序会按照用户的关键词或关键网站来收集相应的信息,并提供给用户想要的信息格式。
            信息量的增加会带来信息网站发布人员工作量的剧增,为实现信息发布系统实现信息自
            动发布、减少工作人员工作量、即时跟踪最新信息,就需要自动信息提供程序,因此Internet信息抓取程序应运而生。
             
            目标

            实现自定义网站信息分类抓取,存入本地数据库、生成静态页面或其它用户定义的信息结构,并下载与信息相关的多媒体文件。
             
            开发
             
            l         目标站点结构分析
            本步骤是准确抓取信息个关键。
            首先要选择更新频率高的页面做为抓取地址,然后分析要抓取内容页面url特点。
            然后分析要抓取信息页面的元素特性,比如标题位置,内容位置 等,得到定位标记点。
            将以上信息写成自己的配置文件或存到数据库中。
            每个网站都需要分析,写出单独的配置文件,供抓取程序使用。
             
            l         信息提取
            根据配置文件取得要抓取页面url,使用HttpWebRequest类获取内容:

            
//获取http页面函数

                    
public string Get_Http(string a_strUrl,int timeout)

                    
{

                        
string strResult ;         

             

                        
try

                        
{

            HttpWebRequest myReq 
= 
            (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;

                            myReq.Timeout 
= timeout;

                            HttpWebResponse HttpWResp 
= 
            (HttpWebResponse)myReq.GetResponse();

                        

                            Stream myStream 
= HttpWResp.GetResponseStream () ;

             

                            StreamReader sr 
= new StreamReader(myStream , 
            Encoding.Default);

                            StringBuilder strBuilder 
= new StringBuilder();

                            
while (-1 != sr.Peek())

                            
{

                                strBuilder.Append(sr.ReadLine()
+" ");

                            }


             

                            strResult 
= strBuilder.ToString();

                        }


                        
catch(Exception exp)

                        
{

                            strResult 
= "错误:" + exp.Message ;

                        }


             

                        
return strResult ;

             

                    }


            获取页面内容后,分析页面中连接地址取到要抓取的url:

            
//处理页面标题和链接

                    
public string SniffWebUrl( string urlStr,string 
            blockB,
string blockE )

                    
{       

                        
string urlch1 = "";

                        
string urlch2 = "";                    

                        
int end_n1 = 0;

                        
int end_nums = 0;

                        
int end_nums1 = 0;

                        
int end_nums2 = 0;

                        
int end_nums3     = 0;            

                        
string reUTStr = "";

                        
string reTitle = "";

                        
string ret = "";           

                        
try

                        
{

                            
int pos01 = urlStr.IndexOf( "." );

                            
int pos02 = urlStr.LastIndexOf( "/" );

                            
if( pos01 < 0 ) 

                            
{

                                
return "";

                            }


                            
if( pos02 < 0 ) 

                            
{

                                
return "";

                            }


                            
int pos03 = urlStr.IndexOf( "/",pos01 );

                            
if ( pos03 < 0 )

                            
{

                                urlch1 
= urlStr;

                                urlch2 
= urlStr;

                            }


                            
else

                            
{

                                urlch1 
= urlStr.Substring( 0,pos03 );

                                urlch2 
= urlStr.Substring( 0,pos02 );

                            }


             

                            
string tmpAllStr = new PublicFun().Get_Http( urlStr 
            ,time1);

             

                            
int pos1 = tmpAllStr.IndexOf( blockB );

                            
int pos2 = tmpAllStr.IndexOf( blockE,pos1 + 
            blockB.Length );

                            
if ( pos1>0 && pos2>0 && pos2>pos1 )

                            
{

                                ret 
= tmpAllStr.Substring( pos1 + 
            blockB.Length,pos2 
- pos1 - blockB.Length );

                                ret 
= ret.Substring( ret.IndexOf( "<" ));

                                
while( ret.IndexOf( "<A" ) >= 0 )

                                
{

                                    ret 
= ret.Substring( 0,ret.IndexOf( "<A" ) ) 
            
+ "<a" + ret.Substring( ret.IndexOf( "<A" ) + 2 );

                                }


                                
while( ret.IndexOf( "</A" ) >=0 )

                                
{

                                    ret 
= ret.Substring( 0,ret.IndexOf( "</A" ) 
            ) 
+ "</a" + ret.Substring( ret.IndexOf( "</A" ) + 3 );

                                }


                                
while( ret.IndexOf( "Href=" ) >=0 )

                                
{

                                    ret 
= ret.Substring( 0,ret.IndexOf( "Href=" 
            )) 
+ "href=" + ret.Substring( ret.IndexOf( "Href=" ) + 5 );

                                }


                                
while( ret.IndexOf( "HREF=" ) >=0 )

                                
{

                                    ret 
= ret.Substring( 0,ret.IndexOf( "HREF=" 
            )) 
+ "href=" + ret.Substring( ret.IndexOf( "HREF=" ) + 5 );

                                }


                                
while( ret.IndexOf( "href='" ) >=0 )

                                
{

                                    ret 
= ret.Substring( 0,ret.IndexOf( "href='" 
            )) 
+ "href="" + ret.Substring( ret.IndexOf( "href='" ) + 6 );

                                }


                            }
       

                            tmpAllStr 
= ret;      

                            
int begin_nums = tmpAllStr.IndexOf( "href=" );

             

                            
while ( begin_nums >= 0 )

                            
{               

                                
string tmpStrA = "";

                                
string tmpStrB = tmpAllStr.Substring( begin_nums 
            
+ 5,1 );

                                
if ( tmpStrB == """ )

                                
{

                                    end_n1 
= begin_nums + 6;

                                    
if ( ( end_n1 + 1 ) > tmpAllStr.Length )

                                    
{

                                        
return "";

                                    }


                                    tmpStrA 
= tmpAllStr.Substring( 
            begin_nums
+6,1 );

                                }


                                
else

                                
{

                                    end_n1 
= begin_nums + 5;

                                    tmpStrA 
= tmpStrB;

                                }


             

                                
if ( tmpStrA == "#" )

                                
{

                                    tmpAllStr 
= tmpAllStr.Substring( end_n1 );

                                    begin_nums 
= tmpAllStr.IndexOf( "href=" );

                                }


                                
else

                                
{                   

                                    end_nums1 
= tmpAllStr.IndexOf( " ",end_n1 );

                                    end_nums2 
= tmpAllStr.IndexOf( ">",end_n1 );

                                    end_nums3 
= tmpAllStr.IndexOf( 
            
"</a",end_nums2 );

             

                                    
if ( ( end_nums3 >= 0 ) && ( end_nums2 >= 0 
            ) )

                                    
{

                                        reTitle 
= tmpAllStr.Substring( end_nums2 
            
+ 1,end_nums3 - end_nums2 - 1 );

             

                                        
if ( end_nums1 > end_nums2 )

                                        
{

                                            end_nums 
= end_nums2;

                                        }


                                        
else

                                        
{

                                            
if ( end_nums1 < 0 )

                                            
{

                                                end_nums 
= end_nums2;

                                            }


                                            
else

                                            
{

                                                end_nums 
= end_nums1;

                                            }


                                        }


                                        
string str4 = tmpAllStr.Substring( 
            end_nums
-1, end_nums - end_nums + 1 );

             

                                        
if ( str4 =="""  || str4 == "'" )

                                        
{

                                            end_nums 
= end_nums - 1;

                                        }


                                        
string sTotalOne = tmpAllStr.Substring( 
            end_n1,end_nums 
- end_n1 );

             

                                        
if ( sTotalOne.IndexOf( "http://" ) <0 )

                                        
{

                                            
if ( sTotalOne.IndexOf( "/" ) == 0 )

                                            
{

                                                sTotalOne 
= urlch1 + sTotalOne;

                                            }


                                            
else

                                            
{                               

                                                
int linshiIntNum = 0;

                                                
int flags = 0;

                                                
string urlChange = urlStr;;

                                                
while( sTotalOne.IndexOf( "../" 
            ) 
>= 0 )

                                                
{

                                                    sTotalOne 
= 
            sTotalOne.Substring( sTotalOne.IndexOf( 
"../" ) + 3 );

                                                    linshiIntNum 
= linshiIntNum 
            
+ 1;

                                                    flags 
= flags +1;

                                                }


                                                
while( ( urlChange.LastIndexOf( 
            
"/" ) >= 0 ) && ( linshiIntNum >= 0 ) )

                                                
{

                                                    urlChange 
= 
            urlChange.Substring( 
0,urlChange.LastIndexOf( "/" ) );

                                                    linshiIntNum 
= linshiIntNum 
            
- 1;

                                                }


                                                
if ( flags == 0 )

                                                
{

                                                    sTotalOne 
= urlch2 + "/" + 
            sTotalOne;

                                                }


                                                
else

                                                
{

                                                    sTotalOne 
= urlChange + "/" 
            
+ sTotalOne;

                                                }


                                            }


                                        }


                                        reUTStr 
= reUTStr + new 
            PublicFun().RemoveHtmlCode( reTitle ) 
+ sTotalOne;

             

                                        tmpAllStr 
= tmpAllStr.Substring( 
            end_nums3 
+ 4 );

                                        begin_nums 
= tmpAllStr.IndexOf( "href=" 
            );

                                    }


                                    
else

                                    
{

                                        begin_nums 
= -1;

                                    }
                    

                                }


                            }


                            
return reUTStr;

                        }


                        
catch( Exception e)

                        
{

                            
return "";

                        }


                    }


             

            得到要抓取内容的url后,处理该页面:

            
//获取链接内容并分类处理

                    
public string GetWebContent( string gatherUrl,string 
            subUrl,
string subTitle,string b_Content,string e_Content,string 
            b_Filter,
string e_Filter,string root )

                    
{

                        
string tmpAllStr = "";            

                        
string dfStrB = "";

                        
string dfStrE = "";                

                        
string rePicStr = "";//图片返回路径    

                        
string reContentStr = "";

                        
string picHtml = "images"//本地图片路径

                        

                        
string urlch1 ="";

                        
string urlch2 ="";

                        
int pos1 = gatherUrl.IndexOf( "." );

                        
int pos2 = gatherUrl.LastIndexOf( "/" );

                        
if( pos1 < 0 )

                        
{

                            
return "";

                        }


                        
if( pos2 < 0 )

                        
{                

                            
return "";

                        }


                        
int pos3 = gatherUrl.IndexOf( "/",pos1 );

                        
if ( pos3 < 0 )

                        
{

                            urlch1 
= gatherUrl;

                            urlch2 
= gatherUrl;

                        }


                        
else

                        
{

                            urlch1 
= gatherUrl.Substring( 0,pos3 );

                            urlch2 
= gatherUrl.Substring( 0,pos2 );

                        }
    

                        

                        tmpAllStr 
= new PublicFun().Get_Http( subUrl,time1 );

                        
//取稿源

                        
string docFromStr = "";

                        
if ( dfStrB != "" && dfStrE != "" )

                        
{

                            
if ( tmpAllStr != "" )

                            
{

                                
int b_docF = tmpAllStr.IndexOf( dfStrB );

                                
if ( b_docF > 0 )

                                
{

                                    
int e_docF = tmpAllStr.IndexOf( 
            dfStrE,b_docF 
+ dfStrB.Length );

                                    
if ( e_docF > 0 && e_docF > b_docF && e_docF 
            
- b_docF < 20 )

                                    
{

                                        docFromStr 
= tmpAllStr.Substring( b_docF 
            
+ dfStrB.Length, e_docF - b_docF - dfStrB.Length );

                                    }


                                }


                            }


                        }


                        
//取内容

                        
if ( tmpAllStr != "" )

                        
{                

                            
int begin_strnum = tmpAllStr.IndexOf( b_Content );

                            
if ( begin_strnum < 0 )

                            
{                   

                                
return "";

                            }


                            
int end_strnum = tmpAllStr.IndexOf( 
            e_Content,begin_strnum 
+ b_Content.Length );

                            
if ( end_strnum < 0 )

                            
{                   

                                
return "";

                            }


                            
string sTotalSubM = "";

                            
if ( end_strnum > begin_strnum )

                            
{

                                sTotalSubM 
= tmpAllStr.Substring ( 
            begin_strnum,end_strnum 
- begin_strnum );

                            }


                            

                            
if ( sTotalSubM == "" )

                            
{                   

                                
return "";

                            }
               

                            
//过滤无用信息

                            
int bfnum = sTotalSubM.IndexOf( b_Filter );

                            
if ( bfnum > -1 )

                            
{

                                
int efnum = sTotalSubM.IndexOf( e_Filter,bfnum 
);

                                
if ( efnum > -1 )

                                
{

                                    
if ( efnum > bfnum )

                                    
{

                                        sTotalSubM 
= sTotalSubM.Substring( 
            
0,bfnum ) + sTotalSubM.Substring( efnum + e_Filter.Length );

                                    }


                                }


                            }


                            
//格式化图片标记

                            

                            
while( sTotalSubM.IndexOf( "Src=" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "Src=" ) ) + "src=" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"Src=" ) + 4 );

                            }


                            
while( sTotalSubM.IndexOf( "SRC=" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "SRC=" ) ) + "src=" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"SRC=" ) + 4 );

                            }


                            
while( sTotalSubM.IndexOf( "src='" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "src='" ) ) + "src="" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"src='" ) + 5 );

                            }


             

                            
//取图片地址

                            
int end_n12 = 0;

                            
int end_nums2 = 0;

                            
int begin_nums2 = sTotalSubM.IndexOf( "src=" );

                            
while( begin_nums2 >= 0 )

                            
{

                                String tmpStr 
= sTotalSubM.Substring( 
            begin_nums2 
+ 4,1 );

                                
if ( tmpStr == """ )

                                
{

                                    end_n12 
= begin_nums2 + 5;

                                }


                                
else

                                
{

                                    end_n12 
= begin_nums2 + 4;

                                }


                                
int end_nums2a = sTotalSubM.IndexOf( " ",end_n12 
            );

                                
int end_nums2b = sTotalSubM.IndexOf( ">",end_n12 
            );

                                
if ( end_nums2b < 0 )

                                
{

                                    
break;

                                }


                                
if ( end_nums2a > end_nums2b )

                                
{

                                    end_nums2 
= end_nums2b;

                                }


                                
else

                                
{

                                    
if (end_nums2a<0)

                                    
{

                                        end_nums2 
= end_nums2b;

                                    }


                                    
else

                                    
{

                                        end_nums2 
= end_nums2a;

                                    }


                                }


                                tmpStr 
= sTotalSubM.Substring( end_nums2-1,1 );

                                
if ( tmpStr == """ || tmpStr == "'" )

                                
{

                                    end_nums2 
= end_nums2 - 1;

                                }


                                
string tmpPicStr = sTotalSubM.Substring( 
            end_n12,end_nums2 
- end_n12 );

             

                                
if ( tmpPicStr.IndexOf( "http://" ) < 0 )

                                
{

                                    
if ( tmpPicStr.IndexOf( "/" ) == 0 )

                                    
{

                                        tmpPicStr 
= urlch1 + tmpPicStr;

                                    }


                                    
else

                                    
{                            

                                        
int linshiIntNum = 0;

                                        
int flags = 0;

                                        
string urlChange = subUrl;

                                        
while( tmpPicStr.IndexOf( "../" ) >= 0 )

                                        
{

                                            tmpPicStr 
= tmpPicStr.Substring( 
            tmpPicStr.IndexOf(
"../"+ 3 );

                                            linshiIntNum 
= linshiIntNum + 1;

                                            flags 
= flags + 1;

                                        }


                                        
while( ( urlChange.LastIndexOf( "/" ) >= 
            
0 ) && ( linshiIntNum >= 0 ) )

                                        
{

                                            urlChange 
= urlChange.Substring( 
            
0,urlChange.LastIndexOf( "/" ) );

                                            linshiIntNum 
= linshiIntNum - 1;

                                        }


                                        
if ( flags == 0 )

                                        
{

                                            tmpPicStr 
= urlch2 + "/" + 
tmpPicStr;

                                        }


                                        
else

                                        
{

                                            tmpPicStr 
= urlChange + "/" + 
            tmpPicStr;

                                        }


                                    }


                                }


                                
//tmpPicStr = tmpPicStr.ToLower();

                                
string tmpPicStrTmp = tmpPicStr.ToLower();

                                
//if ( tmpPicStr.IndexOf( ".jpg" ) > 0 || 
            tmpPicStr.IndexOf( ".gif" ) > 0 || tmpPicStr.IndexOf( ".bmp" ) > 0 )

                                
if ( tmpPicStrTmp.IndexOf( ".jpg" ) > 0 || 
            tmpPicStrTmp.IndexOf( 
".gif" ) > 0 || tmpPicStrTmp.IndexOf( ".bmp" ) 
            
> 0 )

                                
{

                                    rePicStr 
= rePicStr + "||" + tmpPicStr ;

             

                                    
int flagN2 = tmpPicStr.LastIndexOf( "/" );

                                    
string fileN2 = picHtml + 
            tmpPicStr.Substring( flagN2 );

                                    sTotalSubM 
= sTotalSubM.Substring( 
            
0,end_nums2 ) + ">******" + fileN2 + "******<" + 
            sTotalSubM.Substring( end_nums2 );

             

                                    begin_nums2 
= sTotalSubM.IndexOf( "src="
            end_nums2 
+ fileN2.Length + 22 );

                                }


                                
else

                                
{

                                    begin_nums2 
= sTotalSubM.IndexOf( "src="
            end_nums2 
+ 4 );                        

                                }
                   

                            }


                            
if ( rePicStr.Length > 2 )  

                                rePicStr 
=  rePicStr.Substring(2);               


             

                            
//内容处理 格式化关键标记

                            
while( sTotalSubM.IndexOf( "<P" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "<P" ) ) + "|****|<" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"<P" ) + 2 );

                            }


                            
while( sTotalSubM.IndexOf( "<p" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "<p" ) ) + "|****|<" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"<p" ) + 2 );

                            }


                            
while( sTotalSubM.IndexOf( "</P" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "</P" ) ) + "|****|<" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"</P" ) + 3 );

                            }


                            
while( sTotalSubM.IndexOf( "</p" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "</p" ) ) + "|****|<" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"</p" ) + 3 );

                            }


                            
while( sTotalSubM.IndexOf( "<br" ) >=0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "<br" ) ) + "+****+<" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"<br" ) + 3 );

                            }


                            
while( sTotalSubM.IndexOf( "<BR" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "<BR" ) ) + "+****+<" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"<BR" ) + 3 );

                            }


                            
while( sTotalSubM.IndexOf( "<Br" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "<Br" ) ) + "+****+<" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"<Br" ) + 3 );

                            }


                            
while( sTotalSubM.IndexOf( "<bR" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "<bR" ) ) + "+****+<" + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( 
"<bR" ) + 3 );

                            }


             

                            
//去除html标记

                            
int linshiInt1 = sTotalSubM.IndexOf( "<" );

                            
int linshiInt2 = sTotalSubM.IndexOf( ">" );          
              

             

                            
if ( linshiInt2 < linshiInt1 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( linshiInt2 + 
            
1 );

                            }


                            
int linshiInt11 = sTotalSubM.LastIndexOf( "<" );

                            
int linshiInt12 = sTotalSubM.LastIndexOf( ">" );

                            
if ( linshiInt12 < linshiInt11 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 0,linshiInt12 
            
+ 1 );

                            }


                            linshiInt1 
= sTotalSubM.IndexOf( "<" );

                            
while ( linshiInt1 >= 0 )

                            
{

                                linshiInt2 
= sTotalSubM.IndexOf( ">",linshiInt1 
            );

                                
if ( linshiInt2 >= 0 )

                                
{               

                                    sTotalSubM 
= sTotalSubM.Substring( 
            
0,linshiInt1 ) + sTotalSubM.Substring( linshiInt2 + 1 );

                                }


                                
else

                                
{

                                    sTotalSubM 
= sTotalSubM.Substring( 
            
0,linshiInt1 );

                                }


                                linshiInt1 
= sTotalSubM.IndexOf("<");

                            }


             

                            
//还原关键标记

                            
int linshiInt3 = 0;

                            
int linshiInt4 = 0;

             

                            
while( sTotalSubM.IndexOf( "+****+" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "+****+" ) ) + "<br> " + 
            sTotalSubM.Substring( sTotalSubM.IndexOf( 
"+****+" ) + 9 );

                            }


                            
while( sTotalSubM.IndexOf( "|****|" ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( "|****|" ) ) + "<br> " + 
            sTotalSubM.Substring( sTotalSubM.IndexOf( 
"|****|" ) + 9 );

                            }


                            
while( sTotalSubM.IndexOf( "******" ) >= 0 )

                            
{

                                linshiInt3 
= sTotalSubM.IndexOf( "******" ) + 9;

                                linshiInt4 
= sTotalSubM.IndexOf( 
            
"******",linshiInt3 );

                                
if ( linshiInt4 >= 0 )

                                
{

                                    
int tmpPos = sTotalSubM.IndexOf( "******" );

                                    
string tmpStr1 = sTotalSubM.Substring( 
            
0,tmpPos ); 

                                    
string tmpStr2 = sTotalSubM.Substring( 
            linshiInt3,linshiInt4 
- linshiInt3 );

                                    
string tmpStr3 = sTotalSubM.Substring( 
            linshiInt4 
+ 9 );

                                    sTotalSubM 
= tmpStr1 + "<img src=" + tmpStr2 
            
+ ">" + tmpStr3;

                                }


                                
else

                                
{

                                    
break;

                                }


                            }


                            
//去除内容中的标题

                            
if ( sTotalSubM.IndexOf( subTitle ) >= 0 )

                            
{

                                sTotalSubM 
= sTotalSubM.Substring( 
            
0,sTotalSubM.IndexOf( subTitle ) ) + sTotalSubM.Substring( 
            sTotalSubM.IndexOf( subTitle ) 
+ subTitle.Length );

                            }


                            reContentStr 
= sTotalSubM;

                            
//调用下载图片功能

                            
//下载图片到指定目录

                            
string[] img_Url = new PublicFun().split( 
            rePicStr,
"||" );

                            
for ( int i=0;i<img_Url.Length;i++ )

                            
{

                                
if ( img_Url[i] != "" )

                                
{

                                    
new PublicFun().Get_Img( 
            img_Url[i],
10000,root + "/images/" + img_Url[i].Substring( 
            img_Url[i].LastIndexOf(
"/")+1 ) );

                                }


                            }


                        }


                        
return reContentStr;

                    }


            以上方法返回要取得的信息,包括标题内容,图片地址等。

             

            下载页面中图片:

            
//下载图片

                    
public void Get_Img(string a_strUrl,int timeout,string 
            filepath)

                    
{

                        
try

                        
{

            HttpWebRequest myReq 
= 
            (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;

                            myReq.Timeout 
= timeout;

                            HttpWebResponse HttpWResp 
= 
            (HttpWebResponse)myReq.GetResponse();         

                            Stream myStream 
= HttpWResp.GetResponseStream () ;   
                   

                            Bitmap map 
= new Bitmap( myStream );

                            PictureBox picB 
= new PictureBox();

                            picB.Image 
= (Image)map;

                            
string path = filepath.Substring( 
            
0,filepath.LastIndexOf( "/" ) );

                            
if (!Directory.Exists(path))

                            
{

                                CreateDir( path );

                            }
               

                            picB.Image.Save(filepath);                

                        }


                        
catch(Exception exp)

                        
{

                            
string ss = exp.Message;

                    WriteLog( filepath.Substring(
0,filepath.LastIndexOf("/")) + 
            
"/error.log",a_strUrl + "--" + ss + " ");        

                        }


                    }



            l         保存文件或入库
            上面取得的信息可以按自己的要求保存。
             
            
****设计的时候没有使用url按层次循环抓取,这样定义抓取url效率更高,速度更快。

 
0
0

查看评论
* 以上用户言论只代表其个人观点,不代表CSDN网站的观点或立场
    个人资料
    • 访问:109584次
    • 积分:1351
    • 等级:
    • 排名:千里之外
    • 原创:36篇
    • 转载:36篇
    • 译文:0篇
    • 评论:30条
    文章分类
    最新评论