用Jsoup写一个简单的爬虫，并把从网页上面爬下的数据保存到数据库中

今天研究了Jsoup的一些基本用法，来与大家一块分享一下。我是先把从网页上面的数据做成了一个对象，然后把对象存进HashMap中，最后通过JDBC再保存到数据库中。

今天要进行爬虫的网站是智联招聘。每一条招聘信息都可以看成是一个对象。那么就要有一个自定义的JavaBeen类。

其中要用到两个jar包，分别是 ：jsoup-1.10.3.jar 和 mysql-connector-java-5.1.39.jar 两个包。

package javabeen;

import java.util.Date;

/**
 * 工作自定义类
 * 
 * @author xml
 */
public class Job {
	private String position;// 职位
	private String company;// 公司名
	private String compensation;// 薪资
	private String workplace;// 工作地点
	private String date;// 发布日期
	private String education;// 学历
	private String experience;// 工作经验
	private String type;// 职位类别
	private String number;// 工作人数
	private String jobdescription;// 职位描述
	private String comdescription;// 公司描述
	public Job(String position, String company, String compensation, String workplace, String date, String education,
			String experience, String type, String number, String jobdescription, String comdescription) {
		super();
		this.position = position;
		this.company = company;
		this.compensation = compensation;
		this.workplace = workplace;
		this.date = date;
		this.education = education;
		this.experience = experience;
		this.type = type;
		this.number = number;
		this.jobdescription = jobdescription;
		this.comdescription = comdescription;
	}
	public Job() {
		super();
	}
	/**
	 * @return the position
	 */
	public String getPosition() {
		return position;
	}
	/**
	 * @param position the position to set
	 */
	public void setPosition(String position) {
		this.position = position;
	}
	/**
	 * @return the company
	 */
	public String getCompany() {
		return company;
	}
	/**
	 * @param company the company to set
	 */
	public void setCompany(String company) {
		this.company = company;
	}
	/**
	 * @return the compensation
	 */
	public String getCompensation() {
		return compensation;
	}
	/**
	 * @param compensation the compensation to set
	 */
	public void setCompensation(String compensation) {
		this.compensation = compensation;
	}
	/**
	 * @return the workplace
	 */
	public String getWorkplace() {
		return workplace;
	}
	/**
	 * @param workplace the workplace to set
	 */
	public void setWorkplace(String workplace) {
		this.workplace = workplace;
	}
	/**
	 * @return the date
	 */
	public String getDate() {
		return date;
	}
	/**
	 * @param date the date to set
	 */
	public void setDate(String date) {
		this.date = date;
	}
	/**
	 * @return the education
	 */
	public String getEducation() {
		return education;
	}
	/**
	 * @param education the education to set
	 */
	public void setEducation(String education) {
		this.education = education;
	}
	/**
	 * @return the experience
	 */
	public String getExperience() {
		return experience;
	}
	/**
	 * @param experience the experience to set
	 */
	public void setExperience(String experience) {
		this.experience = experience;
	}
	/**
	 * @return the type
	 */
	public String getType() {
		return type;
	}
	/**
	 * @param type the type to set
	 */
	public void setType(String type) {
		this.type = type;
	}
	/**
	 * @return the number
	 */
	public String getNumber() {
		return number;
	}
	/**
	 * @param number the number to set
	 */
	public void setNumber(String number) {
		this.number = number;
	}
	/**
	 * @return the jobdescription
	 */
	public String getJobdescription() {
		return jobdescription;
	}
	/**
	 * @param jobdescription the jobdescription to set
	 */
	public void setJobdescription(String jobdescription) {
		this.jobdescription = jobdescription;
	}
	/**
	 * @return the comdescription
	 */
	public String getComdescription() {
		return comdescription;
	}
	/**
	 * @param comdescription the comdescription to set
	 */
	public void setComdescription(String comdescription) {
		this.comdescription = comdescription;
	}
	/* (non-Javadoc)
	 * @see java.lang.Object#toString()
	 */
	@Override
	public String toString() {
		return "Job [position=" + position + ", company=" + company + ", compensation=" + compensation + ", workplace="
				+ workplace + ", date=" + date + ", education=" + education + ", experience=" + experience + ", type="
				+ type + ", number=" + number + ", jobdescription=" + jobdescription + ", comdescription="
				+ comdescription + "]";
	}

}

然后我们进行爬虫操作

package control;

import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.ListIterator;
import java.util.Set;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


import javabeen.Job;
import utils.datautils;


public class Spider {
	
	static String url = "http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E9%83%91%E5%B7%9E&kw=java&sm=0&p=3";
	static int i = 0;
	public static void body1() throws IOException{
		
        Document doc = Jsoup.connect(url).get();
        
        Element element =  doc.select("div#newlist_list_content_table").first(); 	
//		System.out.println(element);	
        // 进一步获取table元素对应的对象 Elements
		Elements tables = element.select("table");
		HashMap<Integer, Job>hMap = new HashMap<Integer, Job>();
		ListIterator<Element> listIter = tables.listIterator(1 );
		while(listIter.hasNext()){
//			System.out.println(listIter.next());
			
			Element table = listIter.next();
			
			Element link = table.select("tr>td.zwmc").select("a").first();
			Element link1 = table.select("tr>td.gsmc").select("a").first();
			Element link2 = table.select("tr>td.zwyx").first();
			Element link3 = table.select("tr>td.gzdd").first();
			Element link4 = table.select("tr>td.gxsj").select("span").first();
			Element link5 = table.select("tr>td.gxsj").select("span").first();
			Job job = new Job();
			String url1=link.attr("href").toString();
			Document doc1 = Jsoup.connect(url1).get();
			Element element1 =  doc1.select("ul#terminal-ul clearfix").first();
			
//			System.out.println(element1+"===");
			link.text();
			i++;
			
			job.setPosition(link.text().toString());
			job.setCompany(link1.text().toString());
			job.setCompensation(link2.text().toString());
			job.setWorkplace(link3.text().toString());
			job.setDate(link4.text().toString());
			hMap.put(i, job);
		}
		
		Set<Integer> keys = hMap.keySet();
		
		for(Integer key:keys){
			Job value = hMap.get(key);
			Connection conn = null;
			try {
				PreparedStatement ps = null;
				conn = datautils.getConnection();
				String sql = "insert into job(position,company,compensation,workplace,date)values(?,?,?,?,?)";
				ps = conn.prepareStatement(sql);
				ps.setString(1, value.getPosition());
				ps.setString(2, value.getCompany());
				ps.setString(3, value.getCompensation());
				ps.setString(4, value.getWorkplace());
				ps.setString(5, value.getDate());
				ps.executeUpdate();
				conn.close();
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				System.out.println("数据库访问失败");
			}
			System.out.println(key+","+value.toString());
		}
	}
	public static void main(String[] args) throws IOException, ClassNotFoundException, SQLException {
		body1();
	}
}

要访问数据库，所以要用一个数据库个工具类：

package utils;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;


public class datautils {
	 public static final String Driver = "com.mysql.jdbc.Driver";  
	    public static final String url = "jdbc:mysql://localhost:3306/nyb?useSSL=true";  
	    public static final String user = "root";  
	    public static final String password = "123456";  
	  
	    static {  
	        try {  
	            Class.forName(Driver);  
	        } catch (ClassNotFoundException e) {  
	            e.printStackTrace();  
	        }  
	    }  
	    public static Connection getConnection() throws SQLException{  
	        Connection conn = null;  
	        conn = DriverManager.getConnection(url,user,password);  
	        return conn;  
	    }  
}