新浪电子书下载2

package test;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 下载新浪书籍
 * 
 * @author kinkding
 * @history 2009-7-14
 */
public class DownSinaBook {

	public static void main(String[] args) {
		String strUrl = "http://vip.book.sina.com.cn/book/catalog.php?book=79464";
		String filePath = "./tmpdown/";
		DownSinaBook downBook = new DownSinaBook();
		try {
			downBook.startDownload(strUrl, filePath);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

	/**
	 * 开始下载
	 * 
	 * @param filePath
	 * @param bookid
	 * @return
	 * @throws IOException
	 */
	protected int startDownload(String strUrl, String filePath) throws IOException {
		// 下载目录文件
		SimpleDateFormat sdf = new SimpleDateFormat("MMdd-HH-mm-ss");
		String fileName = filePath + sdf.format(new Date()) + ".html";
		downloadFile(strUrl, fileName);
		String rootUrl = strUrl.substring(0, strUrl.lastIndexOf('/') + 1);
		String bookId = strUrl.substring(strUrl.indexOf('=') + 1);

		File f = new File(fileName);
		BufferedReader reader = new BufferedReader(new FileReader(f));
		// <a href="(chapter_79464_\d+\.html)"[^<]*title="(.*)">
		String strPattern = "<a href=\"(chapter_" + bookId + "_\\d+\\.html)\"[^>]*title=\"(.*)\">";
		Pattern pattern = Pattern.compile(strPattern);
		Matcher matcher = pattern.matcher("");

		String line = null;
		Map<String, String> fileMap = new HashMap<String, String>();
		while ((line = reader.readLine()) != null) {
			matcher.reset(line);
			if (matcher.find()) {
				System.out.println(matcher.group(1) + ", " + matcher.group(2));
				fileMap.put(matcher.group(1), matcher.group(2));
			}
		}
		reader.close();
		int total = fileMap.size();
		int i = 0;
		for (String urlName : fileMap.keySet()) {
			String name = fileMap.get(urlName);
			System.out.println(++i + "/" + total + ":" + name);
			this.downloadFile(rootUrl + urlName, filePath + "/" + name);
			this.geneTxt(filePath + "/" + name);
		}
		if (total > 0) {
			f.delete();
		}
		return total;
	}

	/**
	 * 下载文件（返回文件名）
	 * 
	 * @param strUrl
	 * @param filePath
	 * @return
	 * @throws IOException
	 */
	private String downloadFile(String strUrl, String fileName) throws IOException {
		URL url = new URL(strUrl);
		HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
		// 模拟浏览器的访问
		urlConn.setRequestProperty("User-Agent",
				"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.1) Gecko/20090715 Firefox/3.5.1");
		urlConn.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.5");
		// urlConn.setRequestProperty("Accept-Encoding", "gzip,deflate");
		urlConn.setRequestProperty("Accept-Charset", "GB2312,utf-8;q=0.7,*;q=0.7");
		urlConn.setRequestProperty("Keep-Alive", "300");
		urlConn.setRequestProperty("Connection", "keep-alive");
		urlConn.setRequestProperty("Cache-Control", "max-age=0");
		urlConn.setRequestProperty("Referer", strUrl);
		urlConn.connect();
		BufferedInputStream bis = new BufferedInputStream(urlConn.getInputStream());

		FileOutputStream fos = new FileOutputStream(fileName);
		byte[] buf = new byte[1024];
		int len = 0;
		while ((len = bis.read(buf)) != -1) {
			fos.write(buf, 0, len);
		}
		fos.close();
		bis.close();
		urlConn.disconnect();
		return fileName;
	}

	/**
	 * 抽取文本
	 * 
	 * @param fileName
	 * @throws IOException
	 */
	private void geneTxt(String fileName) throws IOException {
		String strPattern = "<div id=\"contTxt\"[^>].*?>(.*)</div>";
		Pattern pattern = Pattern.compile(strPattern, Pattern.DOTALL);
		Matcher matcher = pattern.matcher("");
		File f = new File(fileName);
		BufferedReader reader = new BufferedReader(new FileReader(f));
		String line = null;
		StringBuffer content = new StringBuffer(512);
		while ((line = reader.readLine()) != null) {
			content.append(line.trim());
		}
		reader.close();
		matcher.reset(content);
		String words = "";
		if (matcher.find()) {
			words = matcher.group(1);
		}
		if (words != null && words.length() > 0) {
			f.delete();
			FileWriter fw = new FileWriter(new File(fileName + ".txt"));
			fw.write(words);
			fw.flush();
			fw.close();
		}
	}
}