一个简单的网页抓取例子

package net;


import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class WebContent {
	/**
	 * 读取一个网页全部内容
	 * 
	 * @param htmlurl
	 * @return String 网页内容
	 * @throws IOException
	 */
	public String getOneHtml(final String htmlurl) throws IOException {
		URL url;
		String temp;
		final StringBuffer htmlContent = new StringBuffer();
		try {
			url = new URL(htmlurl);
			System.out.println(url.getProtocol());
			final BufferedReader in = new BufferedReader(new InputStreamReader(
					url.openStream(), "utf-8"));// 读取网页全部内容
			while ((temp = in.readLine()) != null) {
				htmlContent.append(temp);
			}
			in.close();
		} catch (final MalformedURLException me) {
			System.out.println("你输入的URL格式有问题！请仔细输入");
			me.getMessage();
			throw me;
		} catch (final IOException e) {
			e.printStackTrace();
			throw e;
		}
		return htmlContent.toString();
	}


	/**
	 * 
	 * @param s
	 * @return 获得网页标题
	 */
	public String getTitle(final String s) {
		String regex = "<title>.*?</title>";
		;
		String title = "";
		final List<String> list = new ArrayList<String>();
		final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
		final Matcher ma = pa.matcher(s);
		while (ma.find()) {
			list.add(ma.group());
		}
		for (int i = 0; i < list.size(); i++) {
			title = title + list.get(i);
		}
		return outTag(title);
	}


	/**
	 * 
	 * @param s
	 * @return 获得链接
	 */
	public List<String> getLink(final String s) {
		String regex;
		final List<String> list = new ArrayList<String>();
		regex = "<a[^>]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))[^>]*>(.*?)</a>";
		final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
		final Matcher ma = pa.matcher(s);
		while (ma.find()) {
			list.add(ma.group());
		}
		return list;
	}


	/**
	 * 
	 * @param s
	 * @return 获得脚本代码
	 */
	public List<String> getScript(final String s) {
		String regex;
		final List<String> list = new ArrayList<String>();
		regex = "<script.*?</script>";
		final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
		final Matcher ma = pa.matcher(s);
		while (ma.find()) {
			list.add(ma.group());
		}
		return list;
	}


	/**
	 * 
	 * @param s
	 * @return 获得CSS
	 */
	public List<String> getCSS(final String s) {
		String regex;
		final List<String> list = new ArrayList<String>();
		regex = "<style.*?</style>";
		final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
		final Matcher ma = pa.matcher(s);
		while (ma.find()) {
			list.add(ma.group());
		}
		return list;
	}


	/**
	 * 
	 * @param s
	 * @return 去掉标记
	 */
	public String outTag(final String s) {
		return s.replaceAll("<.*?>", "");
	}


	/**
	 * 
	 * @param s
	 * @return
	 */
	public HashMap<String, List<String>> getFromUrls(final String url) {
		final HashMap<String, List<String>> result = new HashMap<String, List<String>>();
		String content = "";
		System.out.println("\n------------------开始读取网页(" + url
				+ ")--------------------");
		try {
			content = getOneHtml(url);
		} catch (final Exception e) {
			e.getMessage();
			return null;
		}
		System.out.println("------------------读取网页(" + url
				+ ")结束--------------------\n");
		System.out.println("------------------分析网页(" + url
				+ ")结果如下--------------------\n");
		List<String> title = new ArrayList<String>();
		title.add(getTitle(content));
		result.put("title", title);
		result.put("css", getCSS(content));
		result.put("script", getScript(content));
		result.put("link", getLink(content));
		return result;
	}


	/**
	 * @param args
	 */
	public static void main(final String args[]) {
		String url = "";
		final List<String> list = new ArrayList<String>();
		System.out.print("输入URL，一行一个，输入结束后输入 go 程序开始运行:   \n");
		final BufferedReader br = new BufferedReader(new InputStreamReader(
				System.in));
		try {
			while (!(url = br.readLine()).equals("go")) // 如果输入不是go那么一直读取
			{
				list.add(url);
			}
		} catch (final Exception e) {
			e.getMessage();
		}
		final WebContent wc = new WebContent();
		HashMap<String, List<String>> hashMap = new HashMap<String, List<String>>();
		for (int i = 0; i < list.size(); i++) {
			if (wc.getFromUrls(list.get(i)) != null) {
				hashMap = wc.getFromUrls(list.get(i));
			}
			for (Iterator<String> iter = hashMap.keySet().iterator(); iter
					.hasNext();) {
				String key = iter.next();
				List<String> list2 = hashMap.get(key);
				System.out.println("--" + key + "内容如下：");
				for (int j = 0; j < list2.size(); j++) {
					System.out.println(list2.get(j));
				}
			}
		}
	}
}