java 网页抓取数据

声明  本文章 属转载

import java.io.BufferedReader;  
import java.io.IOException;   
import java.io.InputStreamReader;   
import java.net.MalformedURLException;   
import java.net.URL;   
import java.util.ArrayList;   
import java.util.HashMap;   
import java.util.Iterator;   
import java.util.List;   
import java.util.regex.Matcher;   
import java.util.regex.Pattern;          
public class WebContent {    
	/** * 
	 * 读取一个网页全部内容       *       
	 * @param htmlurl       
	 * @return String 网页内容       
	 * @throws IOException      
	*/      
   public String getOneHtml(final String htmlurl) throws IOException {
		URL url;         
		String temp;     
		final StringBuffer htmlContent = new StringBuffer();      
		try {              
				url = new URL(htmlurl);     
				System.out.println(url.getProtocol()); //协议  
				final BufferedReader in = 
						new BufferedReader(new InputStreamReader(url.openStream(), "utf-8"));// 读取网页全部内容         
				while ((temp = in.readLine()) != null) { 
					htmlContent.append(temp); 
					System.out.println(htmlContent);
				}               
				in.close();       
		} catch (final MalformedURLException me) {   
				System.out.println("你输入的URL格式有问题！请仔细输入");        
				me.getMessage();      throw me;   
		} catch (final IOException e) {    
	            e.printStackTrace();  throw e;          
        }           
		return htmlContent.toString(); 
}              
	/** 
	* @param s      
    * @return 获得网页标题   
    */ 
	public String getTitle(final String s) {  
		String regex = "<title>.*?</title>";      
		        
		String title = "";     
		final List<String> list = new ArrayList<String>();     
		final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);    
		final Matcher ma = pa.matcher(s);         
		while (ma.find()) {              
			list.add(ma.group());       
		}          
		for (int i = 0; i < list.size(); i++) {  
			title = title + list.get(i);   
		}           
		return outTag(title);  
}              
	/**       
	  * @param s       
	  * @return 获得链接       
	  */ 
	public List<String> getLink(final String s) {  
		String regex;          
		final List<String> list = new ArrayList<String>();  
		regex = "<a[^>]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))[^>]*>(.*?)</a>";   
		final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);     
		final Matcher ma = pa.matcher(s);          
		while (ma.find()) {              
			list.add(ma.group());   
		}          
		return list;    
}             
	/**       
	 * @param s      
	 * @return 获得脚本代码    
   */    
	public List<String> getScript(final String s) {   
		String regex;          
		final List<String> list = new ArrayList<String>();  
		regex = "<script.*?</script>";       
		final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);    
		final Matcher ma = pa.matcher(s);         
		while (ma.find()) {            
			list.add(ma.group());      
		}          
		return list;    
}            
	/**      
	 * @param s      
	 * @return 获得CSS    
	*/  
	public List<String> getCSS(final String s) {  
		String regex;         
		final List<String> list = new ArrayList<String>();   
		regex = "<style.*?</style>";      
		final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);  
		final Matcher ma = pa.matcher(s);       
		while (ma.find()) {             
			list.add(ma.group());      
		}           
		return list;     
}          
	/**      
	 * @param s     
	 * @return 去掉标记      
	*/   
    public String outTag(final String s) {   
    		return s.replaceAll("<.*?>", "");    
 }              
	/**       
	 * @param s    
	 * @return   
     */  
	public HashMap<String, List<String>> getFromUrls(final String url) { 
		final HashMap<String, List<String>> result = new HashMap<String, List<String>>();
		String content = "";        
		System.out.println("\n------------------开始读取网页(" + url   + ")--------------------");   
		try {             
			content = getOneHtml(url);     
		} catch (final Exception e) {      
			e.getMessage();             
			return null;         
		}         
		System.out.println("------------------读取网页(" + url  + ")结束--------------------\n");          
		System.out.println("------------------分析网页(" + url   + ")结果如下--------------------\n");           List<String> title = new ArrayList<String>(); 
		title.add(getTitle(content));   
		result.put("title", title);   
		result.put("css", getCSS(content));  
		result.put("script", getScript(content)); 
		result.put("link", getLink(content));   
		return result;     
}            
	/**      
	 * @param args    
	 */ 
	public static void main(final String args[]) { 
		String url = "";         
		final List<String> list = new ArrayList<String>(); 
		System.out.print("输入URL，一行一个，输入结束后输入 go 程序开始运行:   \n");  
		final BufferedReader br = new BufferedReader(new InputStreamReader(System.in));   
		try {              
			while (!(url = br.readLine()).equals("go")){ // 如果输入不是go那么一直读取        
				list.add(url);      
			}          
		} catch (final Exception e) {    
				e.getMessage();       
		}          
		final WebContent wc = new WebContent(); 
		HashMap<String, List<String>> hashMap = new HashMap<String, List<String>>();   
		for (int i = 0; i < list.size(); i++) {             
			if (wc.getFromUrls(list.get(i)) != null) { 
				hashMap = wc.getFromUrls(list.get(i));  
			}              
			for (Iterator<String> iter = hashMap.keySet().iterator(); iter.hasNext();) {
				String key = iter.next();         
				List<String> list2 = hashMap.get(key);        
				System.out.println("--" + key + "内容如下ssssssssssss：");      
				for (int j = 0; j < list2.size(); j++) {       
					System.out.println(list2.get(j));     
				}             
			}         
		}      
}  
	
}