java爬虫实现_java爬虫简单实现

package WebSpider;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStreamReader;

import java.net.MalformedURLException;

import java.net.URL;

import java.nio.charset.Charset;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

/**

网络爬虫

@author Vcanccc

/

public class WebSpiderTest {

/*

* 获得urlStr对应网络内容

* @param urlStr

* @return

*/

public static String getURLContent(String urlStr, String charset){

StringBuilder sb = new StringBuilder();

try {

URL url = new URL(urlStr);

BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(),Charset.forName(charset)));

String temp = "";

while((temp = reader.readLine()) != null)

{

sb.append(temp);

}

} catch (MalformedURLException e) {

e.printStackTrace();

} catch (IOException e) {

e.printStackTrace();

}

return sb.toString();

}

public static void main(String[] args) {

String destStr = getURLContent("https://www.taobao.com","utf-8");

// Pattern p = Pattern.compile("");

Pattern p = Pattern.compile("href=\"(.+?)\"");

Matcher m = p.matcher(destStr);

while(m.find()){

System.out.println(m.group(1));

}

}

}