简单爬虫的实现原理

import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Test {
 /**
  * *
  *
  * @param args
  */
 public static void main(String[] args) {
  try {
   URL url = new URL("http://www.sina.com.cn");
   URLConnection conn = url.openConnection();
   conn.setDoOutput(true);
   InputStream in = null;
   in = url.openStream();
   String content = pipe(in, "gbk");
   System.out.println(content);
   
   String regex = "<(a|A)(")*>";
   java.util.regex.Pattern pattern = Pattern.compile(regex);
   
   Matcher matcher = pattern.matcher(content);
   
   while(matcher.find()) {
    System.out.println(matcher.group());
    System.out.println(getURL(matcher.group()));
   }
   
   
   
  } catch (Exception e) {
   e.printStackTrace();
  }
 }

 
 public static String getURL(String content) {
  String result = "";
  
  int l = 0;
  int s = 0;
  int e = 0;
  
  l = content.length();
  
  content = content.toLowerCase();
  s = content.indexOf("href=/"") + 6;
  
  e = content.indexOf("/" ", s);
  if(e == -1) {
   e = content.indexOf("/">", s);
  }
  
  if(s > 0 && e > s) {
   result = content.substring(s, e);
  }
  
  return result;
 }
 
 
 public static String pipe(InputStream in, String charset) throws IOException {
  StringBuffer s = new StringBuffer();
  if (charset == null || "".equals(charset)) {
   charset = "utf-8";
  }
  String rLine = null;
  BufferedReader bReader = new BufferedReader(new InputStreamReader(in,
    charset));
  PrintWriter pw = null;
  FileOutputStream fo = new FileOutputStream("../index.html");
  OutputStreamWriter writer = new OutputStreamWriter(fo, "utf-8");
  pw = new PrintWriter(writer);
  while ((rLine = bReader.readLine()) != null) {
   String tmp_rLine = rLine;
   int str_len = tmp_rLine.length();
   if (str_len > 0) {
    s.append(tmp_rLine);
    pw.println(tmp_rLine);
    pw.flush();
    s.append("/n");
   }
   tmp_rLine = null;
  }
  in.close();
  pw.close();
  return s.toString();
 }
}