Java 简单的BFS爬虫

大致意思就是先给定一个URL，然后用HttpParser开源工具提取其网页上的链接，并丢到队列里，然后取队列的首URL重复此操作。

需要用到HttpClient和HttpParser包。

这里先给出所有.Java文件的源码。

其中还有些问题，今天不想看了，所有问题明天再说。

今天做了些修改，还是不能保存到文件，但是应该没有其他的问题了，主要修改了运用Queue来操作，还有修改了提取的Url可能不正确，这里面加了http的判断

-------------------------------------------------------------------------------------MyCrawler.java---------------------------------------------------------------------------------

package MyCrawler;

import java.util.Set;

/**
 *
 * @author xkey
 */
public class MyCrawler {

    /**
     * 使用种子初始化URL队列
     * @return
     * @param seeds 种子URL
     */
    private static void initCrawlerWithSeeds(String[] seeds)
    {
        for(int i = 0 ; i < seeds.length ;  i ++)
        {
            LinkQueue.addUnVisitedUrl(seeds[i]);
        }
    }
    
    /**
     * 抓取过程
     * @return
     * @param seeds
     */
    public static void crawling (String[] seeds)
    {
        initCrawlerWithSeeds(seeds);
        while(!LinkQueue.unVisitedUrlIsEmpty() && LinkQueue.getVisitedUrlNum() < 1000)
        {
            //队头URL
            String visitUrl = (String)LinkQueue.unVisitedUrlDequeue();
            System.out.println(visitUrl);
            if(visitUrl == null) continue;
            DownLoaderFile downLoader = new DownLoaderFile();
            downLoader.downLoaderFile(visitUrl);
            LinkQueue.addVisitedUrl(visitUrl);
            //提取出下载网页中的URL
            LinkFilter filter = new LinkFilter();
            Set<String>links = HtmlParserTool.extracLinks(visitUrl);
            //新的未访问的URL入对
            for(String link:links)
            {
                System.out.println("xkey: "+link);
                LinkQueue.addUnVisitedUrl(link);
            }
            
        }
    }
    public static  void main(String[] args)
    {
        MyCrawler crawler = new MyCrawler();
        crawler.crawling(new String[]{"http://www.baidu.com"});
    }
}

-------------------------------------------------------------------------------------LinkQueue.java------------------------------------------------------------------------------------

public class LinkQueue {
    //已访问的URL集合
    private static Set visitedUrl = new HashSet();
    //待访问的URL集合
    private static Queue <String> unVisitedUrl = new ConcurrentLinkedQueue<String>();
    //获得URL队列
     public static Queue getUnVisitedUrl()
    {
        return unVisitedUrl;
    }
    //添加到访问过的URL队列中
    public static void addVisitedUrl(String Url)
    {
        visitedUrl.add(Url);
    }
    //移除访问过的URL
    public static void removeVisitedUrl(String Url)
    {
        visitedUrl.remove(Url);
    }
    //未访问过的URL出队列
    public static Object unVisitedUrlDequeue()
    {
        return unVisitedUrl.poll();
    }
    //保证每个URL只被访问一次
    public static void addUnVisitedUrl(String Url)
    {
        if(Url != null && !Url.trim().equals("") && !visitedUrl.contains(Url) && !unVisitedUrl.contains(Url))
            unVisitedUrl.add(Url);
    }
    //获得已经访问的URL数目
    public static int getVisitedUrlNum()
    {
        return visitedUrl.size();
    }
    //判断未访问的URL队列是否为空
    public static boolean unVisitedUrlIsEmpty()
    {
        return unVisitedUrl.isEmpty();
    }
}

-------------------------------------------------------------------------------------DownLoaderFile.java---------------------------------------------------------------------------------

public class DownLoaderFile {
    public String getFileNameByUrl(String url,String contentType)
    {
        //移除http
       url = url.substring(7);
       //text.html类型
       if(contentType.indexOf("html") != -1)
       {
           url = url.replaceAll("[\\?/:*|<>\"]","_")+".html";
           return url;
       }
       //pdf类型
       else 
       {
           return url.replaceAll("[\\?/:*|<>\"]","_")+"."+contentType.substring(contentType.lastIndexOf("/")+1);
       }
    }
    /**
     * 保存网页字节数组到本地文件,filepath为要保存的文件相对地址
     */
    private void saveToLocal(byte[] data,String filePath)
    {
        try{
            DataOutputStream out = new DataOutputStream(new FileOutputStream(new File(filePath)));
            for(int i = 0 ; i < data.length ; i ++)
            {
                out.write(data[i]);
            }
            out.flush();
            out.close();
        }catch(IOException e)
        {
            e.printStackTrace();
        }
    }
    //下载URL指向的网页
    public String downLoaderFile(String url)
    {
        String filePath = null;
        //1.生成HttpClient对象并设置参数
        HttpClient httpClient = new HttpClient();
        httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
        //2.生成GetMethod对象并设置参数
        GetMethod getMethod = new GetMethod(url);
        //设置get 请求超时5s
        getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
        getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,new DefaultHttpMethodRetryHandler());
        //3.执行http get qi请求
        try{
            int statusCode  = httpClient.executeMethod(getMethod);
            if(statusCode != HttpStatus.SC_OK)
            {
                System.err.println("Method failed: " + getMethod.getStatusLine());
                filePath = null;
            }
            //4.处理HTTP响应内容
            byte[] responseBody = getMethod.getResponseBody();
            //根据网页URL生成保存时的文件名
           // String ans = new String (responseBody);
           // System.out.println(ans);
            //filePath = "D;/xkey";
          // filePath = "D:\\xkey\\" + getFileNameByUrl(url,getMethod.getResponseHeader("Content-Type").getValue());
           // saveToLocal(responseBody,filePath);
        }catch(HttpException e){
            System.out.println("Please check your provided http address!");
            e.printStackTrace();
        }catch (IOException e)
        {
            e.printStackTrace();
        }finally {
            getMethod.releaseConnection();
        }
        return filePath;
    }
}

-------------------------------------------------------------------------------------HttpParserTool.java--------------------------------------------------------------------------------------------------------------

public class HtmlParserTool {
    //获取一个网站上的URL，filter用来过滤链接
    public static Set<String>extracLinks(String url)
    {
        Set<String>links = new HashSet<String>();
        try{
            Parser parser = new Parser(url);
            parser.setEncoding("utf-8");
            NodeFilter frameFilter = new NodeFilter(){
                public boolean accept(Node node)
                {
                    if(node.getText().startsWith("frame src=")){
                        return true;
                    }
                    else return false;
                }
            };
            
            OrFilter linkFilter = new OrFilter(new NodeClassFilter(LinkTag.class),frameFilter);
            NodeList list = parser.extractAllNodesThatMatch(linkFilter);
            
            for(int i = 0 ; i < list.size() ; i ++)
            {
                Node tag = list.elementAt(i);
                if(tag instanceof LinkTag) //<a>标签
                {
                    LinkTag link = (LinkTag) tag;
                    String linkUrl = link.getLink();//URL
                   if(linkUrl.contains("http"))
                        links.add(linkUrl);
                }else {//<frame>标签
                    //提取frame 里src属性的链接，如<frame src = "test.html">
                    String frame = tag.getText();
                    int start = frame.indexOf("src=");
                    frame = frame.substring(start);
                    int end = frame.indexOf(" ");
                    if(end == -1) end = frame.indexOf(">");
                    String frameUrl = frame.substring(5,end - 1);
                   if(frameUrl.contains("http"))
                            links.add(frameUrl);
                    
                }
            }
        }catch(ParserException e)
        {
            e.printStackTrace();
        }
        return links;
    }
}