java 蜘蛛爬虫_Java网络爬虫蜘蛛

地址：  http://code.google.com/p/crawler4j/

案例：

package com.what21.spider;

import java.util.List;

import java.util.regex.Pattern;

import edu.uci.ics.crawler4j.crawler.Page;

import edu.uci.ics.crawler4j.crawler.WebCrawler;

import edu.uci.ics.crawler4j.parser.HtmlParseData;

import edu.uci.ics.crawler4j.url.WebURL;

public class MyCrawler extends WebCrawler {

private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g"

+ "|png|tiff?|mid|mp2|mp3|mp4"

+ "|wav|avi|mov|mpeg|ram|m4v|pdf"

+ "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

@Override

public boolean shouldVisit(WebURL url) {

String href = url.getURL().toLowerCase();

return !FILTERS.matcher(href).matches() && href.startsWith("http://www.52au3.com/");

}

@Override

public void visit(Page page) {

String url = page.getWebURL().getURL();

System.out.println("URL: " + url);

if (page.getParseData() instanceof HtmlParseData) {

HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();

String text = htmlParseData.getText();

String html = htmlParseData.getHtml();

List links = htmlParseData.getOutgoingUrls();

// 这里可以根据抓取内容做相应的处理

System.out.println("Text length: " + text.length());

System.out.println("Html length: " + html.length());

System.out.println("Number of outgoing links: " + links.size());

}

}

}

package com.what21.spider;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;

import edu.uci.ics.crawler4j.crawler.CrawlController;

import edu.uci.ics.crawler4j.fetcher.PageFetcher;

import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;

import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;

public class Controller {

/**

* @param args

* @throws Exception

*/

public static void main(String[] args) throws Exception {

String crawlStorageFolder = "f:/root";

int numberOfCrawlers = 7;

CrawlConfig config = new CrawlConfig();

config.setCrawlStorageFolder(crawlStorageFolder);

PageFetcher pageFetcher = new PageFetcher(config);

RobotstxtConfig robotstxtConfig = new RobotstxtConfig();

RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);

CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

controller.addSeed("http://www.52au3.com/");

controller.start(MyCrawler.class, numberOfCrawlers);

}

}