java古诗_java抓取古诗文的单线程爬虫

准备知识

1.HTML, CSS, HTML DOM树

参考http://www.w3school.com.cn/htmldom/

2.Jsoup的使用，使用DOM方法遍历一个document对象，使用选择器语法来选择一个元素，从元素中抽取数据。

参考www.open-open.com/jsoup/example-list-links.htm

3.java正则表达式及其语法

参考http://www.cnblogs.com/chuiyuan/p/5187359.html

我们先来做一个单线程的爬虫。

整体步骤：

1.定义爬取内容的对象Poem结构。

2.完成从网上爬取Document对象的模块HttpService功能。

3.从Document对象中解析出所有唐诗的href，并保存到List中。

4.从3中得到的href再爬取出每首古诗的内容。

下面帖一下代码实现。

1.Poem对象只列出其属性

2.抓取Document对象模块HttpService

首先定义一个Rule，封装所有的请求，不管是get还是post。

在HttpService中，使用Rule来抓取document对象。

/**

* Created by chuiyuan on 2/11/16.

*/

public class HttpService {

/**

*

* @param rule

* @return doc

*/

public Document extrace(Rule rule){

validateRule(rule);

String url = rule.getUrl();

String [] params = rule.getParams() ;

String [] values = rule.getValues() ;

String resultTagName = rule.getResultTagName() ;

int type = rule.getType();

int requestMethod = rule.getRequestMethod() ;

Document doc =null ;

try {

Connection conn = Jsoup.connect(url);

conn.userAgent("Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0 Iceweasel/38.5.0");

if (params!= null){

for (int i =0; i

conn.data(params[i], values[i]);

}

}

//Document doc = null ;

switch (requestMethod){

case Rule.GET:

doc =conn.timeout(10000).get();

break;

case Rule.POST:

doc =conn.timeout(10000).post();

break;

}

}catch (IOException e ){

System.out.println("No network");

//e.printStackTrace();

}

return doc ;

}

/**

* validate input params

*/

private static void validateRule(Rule rule){

String url = rule.getUrl() ;

if (url == null || url.length()==0){

throw new RuleException("url can't be null");

}

if (!url.startsWith("http://")){

throw new RuleException("url not in correct format");

}

/**

* not consider total right

*/

if (rule.getParams()!= null && rule.getValues()!= null){

if (rule.getParams().length!= rule.getValues().length){

throw new RuleException("params length!= values length");

}

}

}

}

可以看到，我们先对rule进行了验证，rule中的resultTagName并没有用到，可以去掉。

3.从Document中解析出所有唐诗的href

4.从每首诗文的href中提到详细的内容

public class ProcessDoc {

/**

*

* @param url

* @param httpService

* @return List

*/

public List processGuShiWen(String url, HttpService httpService ){

Rule rule = new Rule(url,

null,null,

null,

-1,

Rule.GET);

Document doc = httpService.extrace(rule);

if (doc == null){

System.out.println("doc null");

return null;

}

//left panel

Elements mainEles = doc.select("div.leftlei");

//String dynasty = mainEles.select("div.son1").first().text();

Elements poemsEles = mainEles.select("div.son2");//size=7 category

//System.out.println(poemsEles.size());

List poemList = new ArrayList();

//Pattern p =Pattern.compile("\\(|\\)");

//some times there is no author

for (Element poemsEle : poemsEles){

Elements poemEles = poemsEle.select("span");

String category1 = poemEles.get(0).text(); //category with ":"

String category = category1.substring(0,category1.length()-1);

for (int i =1;i

Poem poem = new Poem() ;

//poem.setDynasty(dynasty);

poem.setCategory(category);

poem.setHref(poemEles.get(i).select("a").attr("abs:href")+"/");//ref

poem.setTitle(poemEles.get(i).select("a").text());

poemList.add(poem);

}

}

/*for (Poem poem : poemList){

System.out.println(poem.getCategory()+

" "+poem.getTitle()+

" "+poem.getHref());

}*/

return poemList ;

}

/**

* get details of poem

* @param poem

* @param httpService

*/

public void processDetails(Poem poem, HttpService httpService ){

String url = poem.getHref() ;

//System.out.println(url);

Rule rule = new Rule(url,

null,null,

null,

-1,

Rule.GET);

Document doc = httpService.extrace(rule);

if (doc == null) {

System.out.println("doc=null");

return;

}

Elements mainEles = doc.select("div.son2");

//title already ok

Elements poemDetailEles = mainEles.select("p");

String dynasty = poemDetailEles.get(0).text().split("：")[1];//note，chinese :

//Stng dynasty = poemDetailEles.get(0).getElementsByTag("span").text();

//System.out.println(dynasty);

String author = poemDetailEles.get(1).getElementsByTag("a").text();

//System.out.println(author);

String content = mainEles.text().split("原文：")[1];//note

//System.out.println(content);

//do not consider translation

//Element translationEle = doc.select("#")

poem.setDynasty(dynasty);

poem.setAuthor(author);

poem.setContent(content);

//System.out.println(poem.toString());

}

}

整体调用如下

/**

* single thread model,

* save to mysql

*/

public void getGuShiWenSingleThread(){

String url ="http://so.gushiwen.org/gushi/tangshi.aspx/";

HttpService httpService = new HttpService();

ProcessDoc processDoc= new ProcessDoc();

List poemList = processDoc.processGuShiWen(url, httpService);

//get poem content details

for (Poem poem : poemList){

processDoc.processDetails(poem,httpService);

System.out.println(poem.toString());

}

//store to mysql

PoemDao poemDao = new PoemDaoImpl() ;//not PoemDaoImpl

for (Poem poem: poemList){

try {

poemDao.add(poem);

}catch (SQLException e){

e.printStackTrace();

}

}

}

最后结果保存到了MySQL中，数据库部分将在下一篇文章中讲解。