一个简单的email 收集器，base email spider

 最近要在网上收集一些email地址，在网上找了一下工具，结果比较好的都要￥、$，于是便想自己写一个工具：

 

 

 我的思路是这样的：

 

      1.要有一个起始入口，可以使一个特定的搜索结果，也可以是一个网站地址；

 

      2.获取网页内容；

 

      3.在网页内容中提取网页的url（用于继续搜索），提取email地址，使用正则表达式实现；

 

      4.建立3个list，分别保持将要访问的url，已访问的url 和提取成功的email 地址；

 

      5.使用多线程提高效率；

 

代码如下：

 

 

      class Spider { public MyList<string> urls; public MyList<string> emails; public MyList<string> vurls; public string beginUrl; public bool inWebSite=true; private int currentThread = 0; public Spider() { } /// <summary> /// 获取页面的源代码 /// </summary> /// <param name="url"></param> /// <returns></returns> public string GetPageContext(string url) { string result = ""; try { HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(url); myReq.Timeout = 3000; HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse(); Stream myStream = HttpWResp.GetResponseStream(); StreamReader sr = new StreamReader(myStream, Encoding.Default); StringBuilder strBuilder = new StringBuilder(); while (-1 != sr.Peek()) { strBuilder.Append(sr.ReadLine() + " "); } result = strBuilder.ToString(); } catch (Exception ex) { //throw ex; } return result; } /// <summary> /// 在源代码中搜索网址 /// </summary> /// <param name="context"></param> public void SearchUrl(string context,string currentUrl) { Regex rgx = new Regex(@"href=(?<QUOTE>[/""/'])?(?<URL>(?<SCHEME>(file|ftp|http|https|news|nntp):|mailto/:)?(?<EMAIL>[/w-]+@)?(?<HOST>(?(SCHEME)[/w]+(/.[/w-]+)*?))(?<PATH>//?/w*[/w-%/:/./+//]+)?(?<QUERY>/?[/w-%/+:/.]*(=[/w-%/+:/.]*)?(&[/w-%/+/:/.]*(=[/w-%/+:/.]*)?)*)?(?<ANCHOR>/#[/w-%/+:/.]+)?)(?<-QUOTE>[/""/'])?(?#VALIDATE QUOTES/URL)(?(PATH)|(?(SCHEME)|(?!)))(?(QUOTE)(?!))", RegexOptions.Compiled); foreach (Match m in rgx.Matches(context)) { string str = m.ToString().Replace("href=/"", ""); str = str.Replace("/"", ""); if (str.IndexOf("http") < 0) { str = this.beginUrl + str; } if (inWebSite) { if(str.IndexOf(beginUrl) > -1) urls.add(str); } else { urls.add(str); } } SearchEmail(context); } /// <summary> /// 在源代码中搜索email /// </summary> /// <param name="context"></param> public void SearchEmail(string context) { Regex rgx = new Regex(@"/w+([-+.]/w+)*@/w+([-.]/w+)*/./w+([-.]/w+)*", RegexOptions.Compiled); foreach (Match m in rgx.Matches(context)) { if (emails.IndexOf(m.ToString()) < 0) { emails.add(m.ToString()); } } } public void Search() { Search(this.beginUrl); } public void Search(string url) { SearchUrl(GetPageContext(url),url); vurls.add(url); int i = 0; while (urls.Count > 0) { if (vurls.IndexOf(urls[0]) < 0) { if (currentThread < 10) { System.Threading.Thread t = new System.Threading.Thread(ThreadWork); t.Start(); vurls.add(urls[0]); urls.RemoveAt(0); //i++; //if (i > 1000) // break; } } else { urls.RemoveAt(0); } } } private void ThreadWork() { currentThread++; try { SearchUrl(GetPageContext(urls[0]), urls[0]); } catch { } currentThread--; } }

 

 

源代码：