crawler4j爬虫技术获取a标签的title及链接

最近的项目用到了爬虫技术，这里主要说明下crawler4j技术的获取对应的标签的链接及title的实现:

首先是抓取类:
1、必须继承于WebCrawler，实现shouldVisit和visit两个方法。
2、使用一个入口类进行采集任务，并进行处理。

第一点:

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;


import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.lowagie.text.pdf.codec.Base64.InputStream;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

/**
 * 数据采集-爬虫页面处理 shouldVisit: 过滤某个链接是否符合 visit: 读取某个链接内容
 * 
 * @author liuzc
 * @date 2015-2-12
 */
public class GdhhCrawler extends WebCrawler {
    private static final Logger logger = LoggerFactory.getLogger(GdhhCrawler.class);

    /**
     * 过滤二进制文件
     */
    private static final Pattern FILTERS = Pattern
            .compile(".*(\\.(css|js|bmp|gif|jpe?g|png|tiff?|mid|mp2|mp3|mp4|wav|avi|mov|mpeg|ram|m4v|pdf"
                    + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

    private GdhhCrawlController myController = null;
    private DaJob daJob = null;//采集任务
    private Pattern keyWordPattern = null;//关健词形成的正则表达式
    private GdhhCrawlResult crawResult = null;

    public GdhhCrawler() {
        crawResult = new GdhhCrawlResult();
    }

    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {        

        //读取CrawlController 及DaJob
        if(myController == null){
            myController = (GdhhCrawlController)this.getMyController();
            daJob = myController.getDaJob();
        }

        String href = url.getURL().toLowerCase();//链接
        String anchor = url.getAnchor();//标题


        logger.info("href: {}", href);
        logger.info("anchor: {}", anchor);


        //如果采集关健词为空，则按默认规划过滤
        if(daJob == null || daJob.getCrawlKeyWord().equals("")){

            return !FILTERS.matcher(href).matches();
        }

        //如果是种子页面，而且设置为不记录，则不处理
        if(daJob.isSeedUrl(href) /*&& daJob.getFlgStoreSeedPage().equals("N")*/){
            return false;
        }

        //采集关健词不为空，而标题为空，则不采集
        if(NullUtils.isNull(anchor)) 
            return false;

        boolean isMatches;
        if("Y".equals(daJob.getIsGroup())){
            isMatches = getIsMatches(anchor,daJob.getRegPattern());
        }else{
            if(keyWordPattern == null)
                keyWordPattern = Pattern.compile("("+daJob.getCrawlKeyWord()+")");
            isMatches = keyWordPattern.matcher(anchor).find();
        }


        return !FILTERS.matcher(href).matches()
                && isMatches;
    }

    /**
     * 关键词组是否匹配网页锚点
     * author:lhail
     * @param anchor
     * @param regPattern
     * @return
     */
    public boolean getIsMatches(String anchor,Map<String,List<String>> regPattern){
        List<String> listA = regPattern.get("relatedOfKeyWord");//关键词之间的关系
        List<String> listB = regPattern.get("strategy");//关键词策略
        List<String> listC = regPattern.get("keyWords");//关键词
        boolean isMatches = true;
        boolean flag;
        if(!NullUtils.isEmpty(listA)&&"A".equals(listA.get(0))){//与的关系
            for(int i=0;i<listB.size();i++){
                if("0".equals(listB.get(i))){//不包含
                    flag = !Pattern.compile("("+listC.get(i)+")").matcher(anchor).find();
                }else{//包含
                    flag = Pattern.compile("("+listC.get(i)+")").matcher(anchor).find();
                }
                if(!flag){
                    isMatches = false;
                    break;
                }
            }
        }else if(!NullUtils.isEmpty(listA)&&"O".equals(listA.get(0))){//或的关系
            isMatches = false;
            for(int i=0;i<listB.size();i++){
                if("0".equals(listB.get(i))){//不包含
                    flag = !Pattern.compile("("+listC.get(i)+")").matcher(anchor).find();
                }else{//包含
                    flag = Pattern.compile("("+listC.get(i)+")").matcher(anchor).find();
                }
                if(flag){
                    isMatches = true;
                    break;
                }       
            }
        }else{
            isMatches = false;
        }
        return isMatches;
    }


    public void visit(Page page) {
        try{

            //读取CrawlController 及DaJob
            if(myController == null || daJob == null){
                myController = (GdhhCrawlController)this.getMyController();
                daJob = myController.getDaJob();
            }

            WebURL webUrl = page.getWebURL();
            String linkUrl = webUrl.getURL();
            /*
             * 基于数据库的考虑，linkUrl极其有可能超出对应的数据库字段,故限制其对应的长度进入数据库
             * @author lhlong   
             */
            if (linkUrl.length() < 1000) {

                logger.info("Visited: {}", linkUrl);

                SimpleDateFormat dateFormat = new SimpleDateFormat(
                        "yyyy/MM/dd HH:mm:ss");
                Date today = dateFormat.parse(dateFormat.format(new Date()));

                DaResult daResult = new DaResult();

                if (!NullUtils.isNull(daResult.getLinkTitle()))
                    daResult.setUrlname(webUrl.getDomain());

                daResult.setLinkUrl(linkUrl);// 链接URL
                daResult.setCreateTime(today);

                crawResult.incProcessedPages();

                if (page.getParseData() instanceof HtmlParseData) {
                    HtmlParseData parseData = (HtmlParseData) page
                            .getParseData();
                    Set<WebURL> links = parseData.getOutgoingUrls();
                    crawResult.incTotalLinks(links.size());

                    // 下载页面内容另保存为xml文件保存在本地
                    String content = parseData.getHtml();// 链接内容
                    String htmlTitle = parseData.getTitle();//链接的title
                    String tmpTagTitle = modifyHtml(htmlTitle,webUrl.getAnchor()) ; 

                    if(!NullUtils.isNull(tmpTagTitle)){
                        daResult.setLinkTitle(tmpTagTitle);// 链接标题
                    }
                    else 
                        daResult.setLinkTitle(webUrl.getAnchor());// 链接标题

                    // 文件名为：链接地址去掉前缀，后缀，及中间分隔符合
                    String pageName = daResult.getPageName();
                    String filePath = daJob.getCrawlStorageFolder()
                            + File.separator + pageName + ".xml";
                    logger.info("pageName: {}", pageName);
                    // logger.info("content: {}", content);
                    logger.info("filePath: {}", filePath);
                    FileOutputStream out = null;
                    try {
                        out = new FileOutputStream(new File(filePath));
                        out.write(content.getBytes());
                        out.flush();
                    } catch (IOException e) {
                        e.printStackTrace();
                    } finally {
                        try {
                            out.close();
                        } catch (IOException e) {
                        }
                    }
                }

                crawResult.addDaResult(daResult);
            }
        }catch(Exception e){
            e.printStackTrace();
        }
    }
    /**
     * @function 使用jsoup过滤<a>标签里的title属性，找到对应的完整标题
     * @param content
     * @param anchorTmp
     * @return
     * @author lhlong
     * @modify 
     * @date 2015-12-17
     */
    public String modifyHtml(String title,String anchorTmp){

        String [] aTitle ;
        aTitle = title.split("-");
        String tmpTagTitle = "";
        if(!NullUtils.isNull(anchorTmp)){
            //不为空的话，就抓取对应的标题
            if(!NullUtils.isNull(aTitle[0])){
                tmpTagTitle = aTitle[0];
            }
        }
        /*
        try {
            for (WebURL webURL : urls) {
                URL url;
                url = new URL(webURL.getURL());
                Document document;

                document = Jsoup.parse(url, 10000);

                Elements elements = document.getElementsByTag("a");
                for (Element element : elements) {

                        if(!NullUtils.isNull(element.text())){
                            //链接不为空
                            if(element.text().equals(anchorTmp)){
                                if(!NullUtils.isNull(element.attr("title"))){
                                    //title属性不为空
                                    tmpTagTitle = element.attr("title");
                                }
                            }
                        }
                }
            }
        }
        catch (MalformedURLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        catch (IOException e1) {
                // TODO Auto-generated catch block
            e1.printStackTrace();
        }
            */

        return tmpTagTitle;
    }

    /**
     * This function is called by controller to get the local data of this crawler when job is finished
     */
    public Object getMyLocalData() {
        return crawResult;
    }

    /**
     * This function is called by controller before finishing the job. You can
     * put whatever stuff you need here.
     */
    @Override
    public void onBeforeExit() {
        dumpMyData();
    }

    public void dumpMyData() {
        int id = getMyId();
        // You can configure the log to output to file
        logger.info("Crawler {} > Processed Pages: {}", id,
                crawResult.getTotalProcessedPages());
        logger.info("Crawler {} > Total Links Found: {}", id,
                crawResult.getTotalLinks());
        logger.info("Crawler {} > Total Text Size: {}", id,
                crawResult.getTotalTextSize());
    }
}

刚才的这一部分
String content = parseData.getHtml();// 链接内容
String htmlTitle = parseData.getTitle();//链接的title
就是获取当前页面的title属性的方法。

使用这个就能获取到a标签里的title，而不是网页上所截取的标题anchor这个就是网页上显示什么就会展示什么的属性！