网络爬虫基础！

网络爬虫最基础的就是下载文件或页面，然后利用正则表达式提取需要的数据！

假如有如下一个mail.txt文件:需要从里面提取有用的邮箱地址，该怎么做呢？

skdajflksadjflk> abc0@sina.com<djfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajfl> abc1@sina.com<ksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajfl> abc2@sina.com<sadjflkdjfsa;lkdfjks> abc3@sina.com<a;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkv> ab4c@sina.com<dfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajf> abc5@sina.com<ksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjf> abc6@sina.com<lkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkd> abc7@sina.com<fjksa;ljfs;alfj> abc8@sina.com<sa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflks> abc11@sina.com<adjflkdjfsa;> abc9@sina.com<lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjf> abc12@sina.com<lkdjfsa;lk> abc55@sina.com<dfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflk> abc123@sina.com<djfsa;lkdfjksa;ljfs;alfjsa;lkjfs> abc1234@sina.com<a;lkfjska;ljfsafsd
sdfsakfjask;lfjsak;lfj qqq@sina.com aslk;fjsafsafsafs

我们只需要下载这个页面，然后解析其中的邮箱地址即可！

public class TestRegex {
	//获取指定文档中的邮件地址
	public static void getMails() throws IOException {
		BufferedReader bufr = new BufferedReader(new FileReader("D:\\mail.txt"));
		String line = null;
		String mailReg = "\\w+@\\w+(\\.\\w)+";
		Pattern p = Pattern.compile(mailReg);
		
		while((line = bufr.readLine()) != null) {
			Matcher m = p.matcher(line);
			while(m.find()) {
				System.out.println(m.group());
			}
		}
		
	}
	
	public static void main(String[] args) throws IOException {
		getMails();
	}
}

好了，一个简单的网络爬虫就是这样的了！

如果要下载web页面呢？可以使用urlconnection进行下载：

public class TestRegex2 {
	public static void getHtml() throws IOException {
		URL url = new URL("http://s.taobao.com/search?q=%E8%A1%AC%E8%A1%AB&commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&spm=1.7274553.1997520841.1&initiative_id=tbindexz_20150603");
		URLConnection conn = url.openConnection();
		BufferedReader bufferedReader = new BufferedReader(
				new InputStreamReader(conn.getInputStream()));
		String line = null;
//		String mailReg = "g_page_config = ";
//		Pattern p = Pattern.compile(mailReg);
		
		while((line = bufferedReader.readLine()) != null) {
//			Matcher m = p.matcher(line);
//			while(m.find())
//			{
//				System.out.println(m.group());
//			}
			System.out.println(line);
		}
	}
	
	public static void main(String[] args) throws IOException {
		getHtml();
	}
}