java爬虫学习

1.新建java项目，导入需要用到的jar包：
jsoup-1.8.1.jar
junit-4.8.2.jar
2.新建Rule.java

public class Rule {
	 /** 
     * 链接 
     */  
    private String url;  
  
    /** 
     * 参数集合 
     */  
    private String[] params;  
    /** 
     * 参数对应的值 
     */  
    private String[] values;  
  
    /** 
     * 对返回的HTML，第一次过滤所用的标签，请先设置type 
     */  
    private String resultTagName;  
  
    /** 
     * CLASS / ID / SELECTION 
     * 设置resultTagName的类型，默认为ID  
     */  
    private int type = ID ;  
      
    /** 
     *GET / POST 
     * 请求的类型，默认GET 
     */  
    private int requestMoethod = GET ;   
      
    public final static int GET = 0 ;  
    public final static int POST = 1 ;  
      
  
    public final static int CLASS = 0;  
    public final static int ID = 1;  
    public final static int SELECTION = 2;  
  
    public Rule()  
    {  
    }  
  
      
    public Rule(String url, String[] params, String[] values,  
            String resultTagName, int type, int requestMoethod)  
    {  
        super();  
        this.url = url;  
        this.params = params;  
        this.values = values;  
        this.resultTagName = resultTagName;  
        this.type = type;  
        this.requestMoethod = requestMoethod;  
    }  
  
    public String getUrl()  
    {  
        return url;  
    }  
  
    public void setUrl(String url)  
    {  
        this.url = url;  
    }  
  
    public String[] getParams()  
    {  
        return params;  
    }  
  
    public void setParams(String[] params)  
    {  
        this.params = params;  
    }  
  
    public String[] getValues()  
    {  
        return values;  
    }  
  
    public void setValues(String[] values)  
    {  
        this.values = values;  
    }  
  
    public String getResultTagName()  
    {  
        return resultTagName;  
    }  
  
    public void setResultTagName(String resultTagName)  
    {  
        this.resultTagName = resultTagName;  
    }  
  
    public int getType()  
    {  
        return type;  
    }  
  
    public void setType(int type)  
    {  
        this.type = type;  
    }  
  
    public int getRequestMoethod()  
    {  
        return requestMoethod;  
    }  
  
    public void setRequestMoethod(int requestMoethod)  
    {  
        this.requestMoethod = requestMoethod;  
    }  
}

3.新建LinkTypeData.java

public class LinkTypeData {
	 private int id;  
	    /** 
	     * 链接的地址 
	     */  
	    private String linkHref;  
	    /** 
	     * 链接的标题 
	     */  
	    private String linkText;  
	    /** 
	     * 摘要 
	     */  
	    private String summary;  
	    /** 
	     * 内容 
	     */  
	    private String content;  
	    public int getId()  
	    {  
	        return id;  
	    }  
	    public void setId(int id)  
	    {  
	        this.id = id;  
	    }  
	    public String getLinkHref()  
	    {  
	        return linkHref;  
	    }  
	    public void setLinkHref(String linkHref)  
	    {  
	        this.linkHref = linkHref;  
	    }  
	    public String getLinkText()  
	    {  
	        return linkText;  
	    }  
	    public void setLinkText(String linkText)  
	    {  
	        this.linkText = linkText;  
	    }  
	    public String getSummary()  
	    {  
	        return summary;  
	    }  
	    public void setSummary(String summary)  
	    {  
	        this.summary = summary;  
	    }  
	    public String getContent()  
	    {  
	        return content;  
	    }  
	    public void setContent(String content)  
	    {  
	        this.content = content;  
	    }  
}

4.新建TextUtil.java工具类

public class TextUtil {
	public static boolean isEmpty(String url){
		if (url == null) {
			return true;
		} else if (url.toLowerCase().equals("null")) {
			return true;
		}else if(url.equals("")){
			return true;
		}else{
			return false;
		}
	}
}

5.新建RuleException.java异常类

public class RuleException extends RuntimeException{
	 public RuleException()  
	    {  
	        super();  
	        // TODO Auto-generated constructor stub  
	    }  
	  
	    public RuleException(String message, Throwable cause)  
	    {  
	        super(message, cause);  
	        // TODO Auto-generated constructor stub  
	    }  
	  
	    public RuleException(String message)  
	    {  
	        super(message);  
	        // TODO Auto-generated constructor stub  
	    }  
	  
	    public RuleException(Throwable cause)  
	    {  
	        super(cause);  
	        // TODO Auto-generated constructor stub  
	    }  
	  
}

6.ExtractService.java

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.jsoup.Connection;
import org.jsoup.Jsoup;  
import org.jsoup.nodes.Document;  
import org.jsoup.nodes.Element;  
import org.jsoup.select.Elements;  


public class ExtractService {
	 /** 
     * @param rule 
     * @return 
     */  
    public static List<LinkTypeData> extract(Rule rule)  
    {  
        // 进行对rule的必要校验  
        validateRule(rule);  
  
        List<LinkTypeData> datas = new ArrayList<LinkTypeData>();  
        LinkTypeData data = null;  
        try  
        {  
            /** 
             * 解析rule 
             */  
            String url = rule.getUrl();  
            String[] params = rule.getParams();  
            String[] values = rule.getValues();  
            String resultTagName = rule.getResultTagName();  
            int type = rule.getType();  
            int requestType = rule.getRequestMoethod();  
  
            Connection conn = Jsoup.connect(url).ignoreContentType(true);
            //Document pod = Jsoup.connect(url).ignoreContentType(true).get();
            // 设置查询参数  
  
            if (params != null)  
            {  
                for (int i = 0; i < params.length; i++)  
                {  
                    conn.data(params[i], values[i]);  
                }  
            }  
  
            // 设置请求类型  
            Document doc = null;  
            switch (requestType)  
            {  
            case Rule.GET:  
                doc = conn.timeout(100000).get();  
                break;  
            case Rule.POST:  
                doc = conn.timeout(100000).post();  
                break;  
            }  
  
            //处理返回数据  
            Elements results = new Elements();  
            switch (type)  
            {  
            case Rule.CLASS:  
                results = doc.getElementsByClass(resultTagName);  
                break;  
            case Rule.ID:  
                Element result = doc.getElementById(resultTagName);  
                results.add(result);  
                break;  
            case Rule.SELECTION:  
                results = doc.select(resultTagName);  
                break;  
            default:  
                //当resultTagName为空时默认去body标签  
                if (TextUtil.isEmpty(resultTagName))  
                {  
                    results = doc.getElementsByTag("body");  
                }  
            }  
  
            for (Element result : results)  
            {  
//                Elements links = result.getElementsByTag("a");  
//  
//                for (Element link : links)  
//                {  
//                    //必要的筛选  
//                    String linkHref = link.attr("href");  
//                    String linkText = link.text();  
//  
//                    data = new LinkTypeData();  
//                    data.setLinkHref(linkHref);  
//                    data.setLinkText(linkText);  
//  
//                    datas.add(data);  
//                }  
            	Elements links = result.getElementsByTag("body");
            	for (Element link : links) {
            		String linkHref = link.attr("href");  
	                String linkText = link.text();  
	                data = new LinkTypeData();  
	                data.setLinkHref(linkHref);  
	                data.setLinkText(linkText);  
	                datas.add(data);  
				}
            }  
  
        } catch (IOException e)  
        {  
            e.printStackTrace();  
        }  
        return datas;  
    }  
  
    /** 
     * 对传入的参数进行必要的校验 
     */  
    private static void validateRule(Rule rule)  
    {  
        String url = rule.getUrl();  
        if (TextUtil.isEmpty(url))  
        {  
            throw new RuleException("url不能为空！");  
        }  
        if (!url.startsWith("http://"))  
        {  
            throw new RuleException("url的格式不正确！");  
        }  
  
        if (rule.getParams() != null && rule.getValues() != null)  
        {  
            if (rule.getParams().length != rule.getValues().length)  
            {  
                throw new RuleException("参数的键值对个数不匹配！");  
            }  
        }  
  
    }  
}

7.测试类（junit单元测试）

import java.util.List;

public class Test {
	 	@org.junit.Test  
	    public void getDatasByClass()  
	    {  
//	        Rule rule = new Rule(  
//	                "http://www1.sxcredit.gov.cn/public/infocomquery.do?method=publicIndexQuery",  
//	        new String[] { "query.enterprisename","query.registationnumber" }, new String[] { "兴网","" },  
//	                "cont_right", Rule.CLASS, Rule.POST);  
	        
	        Rule rule=new Rule("http://hiweshare.com/topicaction/gettopics.do", new String[]{"page","rows"}, new String[]{"1","12"}, null, -1, Rule.POST);
	        List<LinkTypeData> extracts = ExtractService.extract(rule);  
	        printf(extracts);  
	    }  
	 
	 public void printf(List<LinkTypeData> datas)  
	    {  
	        for (LinkTypeData data : datas)  
	        {  
	            System.out.println(data.getLinkText());  
	            System.out.println(data.getLinkHref());  
	            System.out.println("***********************************");  
	        }  
	    }  
}

8.可能会出现的报错信息：
如果这样获取连接的时候可能会报错：

Connection conn = Jsoup.connect(url)

报错信息：

Unhandled content type. Must be text/*, application/xml, or application/xhtm

修改为：

Connection conn = Jsoup.connect(url).ignoreContentType(true);

hiweshare.com是我的一个正在建设的网站，我先在上面测试了下爬虫

http://hiweshare.com/

感谢博主：http://blog.csdn.net/lmj623565791/article/details/23272657