使用htmlparser简单抓取京东图书信息存入数据库的小例子

在学习lucene的时候需要很大的数据，很大的数据怎么来，不可能手工一点输入，从网上找了个方法，抓取数据。

我把目标锁定在了京东商城的图书信息。下面是我抓取图书信息的一个简单例子，写的不是特别严谨，但对我学习来说足够了。如果每个页面都有效的话完全能达到80万的数据。我把抓到的信息存入我设计的数据库中。在处理过程中，把处理好的网址信息以及出错信息存入e:/360book/book1.txt，使用前在E盘创建一个360book的文件夹。

数据库表结构为

CREATE TABLE `books` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
   `bookName` varchar(1024) DEFAULT NULL,
   `auther` varchar(1024) DEFAULT NULL,
   `booksName` varchar(1024) DEFAULT NULL,
   `publish` varchar(1024) DEFAULT NULL,
   `isbn` varchar(1024) DEFAULT NULL,
   `publishDate` varchar(1024) DEFAULT NULL,
   `edition` varchar(1024) DEFAULT NULL,
   `pages` varchar(1024) DEFAULT NULL,
   `frame` varchar(1024) DEFAULT NULL,
   `format` varchar(1024) DEFAULT NULL,
   `type` varchar(1024) DEFAULT NULL,
   `price` varchar(1024) DEFAULT NULL,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB AUTO_INCREMENT=15779 DEFAULT CHARSET=utf8

例子中使用的插件是htmlparser，版本为2.0。

这是处理页面信息的一个Class.

package com;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.HashMap;
import java.util.Map;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.HasChildFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.util.NodeList;

public class Buy {

	public static void main(String []args) throws Exception {
		FileWriter fileWriter = null;
		PrintWriter printWriter = null;
		File file = new File("e:/360book/book1.txt");
		for(int i=2;i<=861852;i++){
			int t = 10000000 + i;
			String url = "http://book.360buy.com/"+t+".html";
			//System.out.println(url);
			try {
				Buy.getBook(url);
				fileWriter = new FileWriter(file,true);
				printWriter = new PrintWriter(fileWriter);
				printWriter.println(url+"\t is ok!");
				printWriter.close();
				fileWriter.close();
			} catch (Exception e) {
				e.printStackTrace();
				try {
					fileWriter = new FileWriter(file,true);
					printWriter = new PrintWriter(fileWriter);
					printWriter.println(e);
					printWriter.close();
					fileWriter.close();
				} catch (IOException e1) {
					e1.printStackTrace();
				}
			}
		}
	}
	
	public static void getBook(String url) throws Exception {
		
		Parser parser = new Parser((HttpURLConnection)new URL(url).openConnection());
		parser.setEncoding("gb2312");
		NodeFilter nameFilter = new HasAttributeFilter("id","name");
		NodeList nodeList = parser.extractAllNodesThatMatch(nameFilter);
		String bookName = getBookName(nodeList);
		
		NodeFilter infoFilter = new HasAttributeFilter("id","summary");
		parser.reset();
		NodeList summaryList = parser.extractAllNodesThatMatch(infoFilter);
		Map<String,String> map = getSummary(summaryList);
		
		NodeFilter priceFilter = new HasAttributeFilter("id","book-price");
		parser.reset();
		NodeList priceList = parser.extractAllNodesThatMatch(priceFilter);
		String price = getPrice(priceList);
		Connection con = DBCon.getConnection();
		save(con,bookName,map,price);
		DBCon.closeConnection(con);
	}
	
	public static void save(Connection con ,String bookName,Map<String,String> map,String price)throws Exception {
		PreparedStatement pstmt = null;
		String sql = "insert into books (bookName,auther,booksName,publish,isbn,publishDate,edition,pages,frame,format,type,price) values(?,?,?,?,?,?,?,?,?,?,?,?)";
		pstmt = con.prepareStatement(sql);
		pstmt.setString(1, bookName);
		pstmt.setString(2, map.get("auther"));
		pstmt.setString(3, map.get("booksName"));
		pstmt.setString(4, map.get("publish"));
		pstmt.setString(5, map.get("isbn"));
		pstmt.setString(6, map.get("publishDate"));
		pstmt.setString(7, map.get("edition"));
		pstmt.setString(8, map.get("pages"));
		pstmt.setString(9, map.get("frame"));
		pstmt.setString(10, map.get("format"));
		pstmt.setString(11, map.get("type"));
		pstmt.setString(12, price);
		pstmt.executeUpdate();
		DBCon.closePreparedStatement(pstmt);
	}
	
	public static Map<String,String> getSummary(NodeList nodeList){
		
		Map<String,String> map = new HashMap<String,String>();
		nodeList = nodeList.elementAt(0).getChildren().extractAllNodesThatMatch(new TagNameFilter("li"));
		for(int i=0;i<nodeList.size();i++){
			switch(i){
			case 0:map.put("auther", getAuther(nodeList.elementAt(i)));break;
			case 1:map.put("booksName", getValue(nodeList.elementAt(i)));break;
			case 2:map.put("publish", getPublish(nodeList.elementAt(i)));break;
			case 3:map.put("isbn", getValue(nodeList.elementAt(i)));break;
			case 4:map.put("publishDate", getValue(nodeList.elementAt(i)));break;
			case 5:map.put("edition", getValue(nodeList.elementAt(i)));break;
			case 6:map.put("pages", getValue(nodeList.elementAt(i)));break;
			case 7:map.put("frame", getValue(nodeList.elementAt(i)));break;
			case 8:map.put("format", getValue(nodeList.elementAt(i)));break;
			case 9:map.put("type", getPublish(nodeList.elementAt(i)));break;
			}
		}
		
		return map;
	}
	
	public static String getValue(Node node){
		
		String temp = node.toHtml();
		return temp.substring(temp.indexOf("</span>")+7, temp.indexOf("</li>"));
	}
	
	public static String getPublish(Node node){
		
		String publish = "";
		NodeFilter nodeFilter = new TagNameFilter("a");
		NodeList nodeList = node.getChildren().extractAllNodesThatMatch(nodeFilter);
		if(nodeList!=null){
			for(int i=0;i<nodeList.size();i++){
				publish = publish + nodeList.elementAt(i).getChildren().elementAt(0).getText();
				if(i!=nodeList.size()-1){
					publish = publish + ",";
				}
			}
		}
		return publish;
	}
	
	public static String getAuther(Node node){
		
		String auther = ""; 
		NodeFilter nodeFilter = new TagNameFilter("a");
		NodeList nodeList = node.getChildren().extractAllNodesThatMatch(nodeFilter);
		if(nodeList!=null){
			for(int i=0;i<nodeList.size();i++){
				auther = auther + nodeList.elementAt(i).getChildren().elementAt(0).getText();
				if(i!=nodeList.size()-1){
					auther = auther +",";
				}
			}
		}
		return auther;
	}
	
	public static String getBookName(NodeList nodeList){
		
		String bookName = "";
		String temp = nodeList.elementAt(0).getChildren().toHtml();
		bookName = temp.substring(temp.indexOf("<h1>")+4, temp.indexOf("<span>"));
		
		return bookName;
	}
	
	public static String getPrice(NodeList nodeList){
		
		String price = "";
		NodeFilter childFilter = new TagNameFilter("del");
		NodeFilter nodeFilter = new HasChildFilter(childFilter);
		nodeList = nodeList.elementAt(0).getChildren().extractAllNodesThatMatch(nodeFilter);
		String temp = nodeList.toHtml();
		price = temp.substring(temp.indexOf("<del>")+5, temp.indexOf("</del>"));
		
		return price;
	}
}

DBCon.java

package com;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

//import javax.naming.Context;
//import javax.naming.InitialContext;
//import javax.sql.DataSource;

public class DBCon {
	public static Connection getConnection(){
		Connection con=null;
		//Context ctx;
		try {
			//ctx = new InitialContext();
			//DataSource ds = (DataSource) ctx.lookup("java:comp/env/jdbc/myhelp");
			//con = ds.getConnection();
			Class.forName("com.mysql.jdbc.Driver").newInstance();
			con=DriverManager.getConnection("jdbc:mysql://localhost:3306/test?user=root&password=root");
		} catch (Exception e) {
			e.printStackTrace();
		}
		return con;
	}
	
	public static void closeConnection(Connection con) {
		if(con!=null){
			try {
				con.close();
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
	}
	
	public static void closeResultSet(ResultSet rs) {
		if(rs!=null){
			try {
				rs.close();
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
	}
	
	public static void closePreparedStatement(PreparedStatement pstmt){
		if(pstmt!=null){
			try {
				pstmt.close();
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
	}
	
	public static void closeStatement(Statement stmt){
		if(stmt!=null){
			try {
				stmt.close();
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
	}
}