写个爬虫获取全国地区数据

公司需要全国的地址数据，网上找半天找不到最后找到这个网址，怎么办?咱是程序猿，直接写个爬虫搞起！

看看测试效果

public static void main(String[] args) throws IOException {
		
		//构建一个网页处理器
		PageProcessor processor = new NetworkProcessor();
		//构建两个数据提取器，专门用来提取城市地址数据
		AnalyHandler analy = new CityUrlExtractHandler();
		analy.setAnalyHandler(new CityNameExtractHandler());
		//构建一个网页数据过滤器
		PageFilter filter = new DefaultPageFilter();
		
		//构建一个网页对象
		Page page = new Page();
		page.setRegex("<td>.*?</td>");//整个网页数据提取规则
		page.setUrl("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2015/index.html");//需要爬取的url
		page.setLevel(1);	//页面深度第一次是1
		
		//构建一个爬虫启动对象
		InvokeCenter ic = new InvokeCenter(processor, analy);
		ic.setPageFilter(filter);
		
		//取得爬取结果
		List<Map<String,String>> result = ic.start(page);
		String fileDir = "C:\\Users\\Administrator\\Desktop\\cityData.txt";
		
		fullDisk(result,fileDir);
	}

花了两个多小时写的，感觉还不错，只有InvokeCenter爬虫启动中心还需要做修改，让其可以支撑各种网页的爬取，这个我就不做了，如果你拿到了我的代码，你可以去改下试试！

好了，贴上全部代码！

package com.fyrj.compoment.crawler;

/***
 * 代表一个抽象网页
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public class Page {
	
	/***
	 * 网页URL
	 */
	private String url ; 
	
	/***
	 * 页面数据
	 */
	private String viewData;
	
	/***
	 * 该页面提取数据的规则表达式
	 */
	private String regex;
	
	/***
	 * 页面深度
	 */
	private int level;

	public String getUrl() {
		return url;
	}

	public void setUrl(String url) {
		this.url = url;
	}

	public String getViewData() {
		return viewData;
	}

	public void setViewData(String viewData) {
		this.viewData = viewData;
	}

	public String getRegex() {
		return regex;
	}

	public void setRegex(String regex) {
		this.regex = regex;
	}

	public int getLevel() {
		return level;
	}

	public void setLevel(int level) {
		this.level = level;
	}
	
	
}

package com.fyrj.compoment.crawler;

import java.util.UUID;

/***
 * 静态编码以及ID生成器
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public class NumberAndIdGenerator {
	private static int number = 100001;
	
	private NumberAndIdGenerator() {}
	
	/***
	 * 采用静态内部类实现单列
	 * @author ying.cai
	 * @email 919126624@qq.com
	 * @version 1.0
	 */
	public static class Inner{
		private static NumberAndIdGenerator instance = new NumberAndIdGenerator();
	}
	
	public static NumberAndIdGenerator getInstence(){
		return Inner.instance;
	}
	/***
	 * 生成编码
	 * @return
	 */
	public static String createNumber(){
		return ++number+"";
	}
	
	/***
	 * 生成Id
	 * @return
	 */
	public static String createId(){
		return UUID.randomUUID().toString().replaceAll("-", "");
	}
	
}

package com.fyrj.compoment.crawler;

/***
 * 页面数据获取接口
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public interface PageProcessor {
	
	Page resolverPage(Page page);

}

package com.fyrj.compoment.crawler;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

/***
 * 通过网络获取页面数据
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public class NetworkProcessor implements PageProcessor {

	@Override
	public Page resolverPage(Page page) {
		String urlStr = page.getUrl();
		if( urlStr == null || urlStr.equals("")){
			throw new RuntimeException("page url not be null!");
		}
		try{
			
			URL url = new URL(urlStr);
			HttpURLConnection urlConnection = (HttpURLConnection)url.openConnection();
			int responsecode=urlConnection.getResponseCode();
			BufferedReader reader;
			StringBuffer bufferData = new StringBuffer();
	        if(responsecode==200){
	            reader=new BufferedReader(new InputStreamReader(urlConnection.getInputStream(),"GBK"));
	            String lineData = null;
	    		while( null!=(lineData=reader.readLine())){
	    			bufferData.append(lineData);
	    		}
	        }else{
	        	throw new IOException("获取不到网页的源码，服务器响应代码为："+responsecode+"URL为："+urlStr);
	        }
	        page.setViewData(bufferData.toString());
		}catch(Exception e){
	    	throw new RuntimeException("获取不到网页的源码,出现异常：",e);
	    }
	    
        return page;
	}
	
}

package com.fyrj.compoment.crawler;

import java.util.Map;

/***
 * 动态匹配规则中心
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public interface RuleDefinition {
	
	/***
	 * 规则定义
	 * @return
	 */
	String getRegex();
	
	/***
	 * 争对拿到的数据进行解析处理
	 * @param lineData
	 * @param result
	 * @return
	 */
	boolean fullData( String lineData, Map<String,String> result );
}

package com.fyrj.compoment.crawler;

/***
 * 页面解析过滤器
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public abstract class  PageFilter {
	protected PageFilter pageFilter ; 
	
	public Page doFilter(Page page){
		this.filter(page);
		if(this.pageFilter!=null){
			pageFilter.filter(page);
		}
		return page;
	};
	
	protected abstract void filter(Page page);
}

package com.fyrj.compoment.crawler;

/***
 * 针对地址解析这个网站得有这么个过滤器
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public class DefaultPageFilter extends PageFilter {

	@Override
	public void filter(Page page) {
		if( page.getLevel() >1 ){
			page.setRegex("<td>.*?</td><td>.*?</td>");
			page.setViewData(page.getViewData()
					.replaceAll("<td>名称</td>", "")
					.replaceAll("<td>\\D</td>", "")
					);
		}
		
	}

}

package com.fyrj.compoment.crawler;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/***
 * 数据提取器
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public abstract class AnalyHandler {
	
	protected String dataStr;
	protected AnalyHandler analyHandler ; 
	
	//采用链条式一直调用处理者直到处理完毕
	public Map<String,String> analyStart(){
		Map<String,String> result = new HashMap<>();
		analyDefine(result);
		if( this.analyHandler!=null ){
			analyHandler.setDataStr(dataStr);
			analyHandler.analyDefine(result);
		}
		return result;
	}
	
	protected Map<String,String> analyDefine( Map<String,String> result ){
		//这里采用规则链条调用...直到能匹配出数据！或者最后都匹配不到数据那就放弃算了！
		List<RuleDefinition> rList = getRuleDefinitionChain();
		for (RuleDefinition ruleDefinition : rList) {
			String regex = ruleDefinition.getRegex();
			List<String> list = new ArrayList<String>();
			Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
			Matcher ma = pa.matcher(dataStr);
			while (ma.find()){
				list.add(ma.group());
			}
			if(list.size()>0){
				String lineData = list.get(0);
				boolean fullResult = ruleDefinition.fullData(lineData, result);
				//如果解析成功拿到了东西，就不再继续往下解析，否则继续到第二个规则对象中去解析数据！
				if(fullResult){
					return result;
				}
			}
		}
		return result;
	}
	
	
	public void setAnalyHandler( AnalyHandler analyHandler){
		this.analyHandler = analyHandler;
	}
	
	public void setDataStr( String dataStr){
		this.dataStr = dataStr;
	};

	abstract List<RuleDefinition> getRuleDefinitionChain();
}

package com.fyrj.compoment.crawler;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/***
 * 解析出城市名称
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public class CityNameExtractHandler extends AnalyHandler{

	@Override
	List<RuleDefinition> getRuleDefinitionChain() {
		List<RuleDefinition> rList = new ArrayList<>();
		//第一规则，这个争对第一级地址可用
		rList.add(new RuleDefinition() {
			@Override
			public String getRegex() {
				return "html'>.*?<br/>";
			}
			
			@Override
			public boolean fullData(String lineData, Map<String, String> result) {
				String cityName = lineData.substring(lineData.lastIndexOf("html'>")+6,lineData.lastIndexOf("<br/>"));;
				if(cityName==null || "".equals(cityName)){
					//如果没解析到数据，证明这次是失败的！
					return false;
				}
				result.put("CITY_NAME", cityName);
				return true;
			}
		});
		
		//第二规则，这个针对中间地址可用
		rList.add(new RuleDefinition() {
			@Override
			public String getRegex() {
				return "html'>\\D*?</a>";
			}
			
			@Override
			public boolean fullData(String lineData, Map<String, String> result) {
				String cityName = lineData.substring(lineData.lastIndexOf("html'>")+6,lineData.lastIndexOf("</a>"));;
				if(cityName==null || "".equals(cityName)){
					//如果没解析到数据，证明这次是失败的！
					return false;
				}
				result.put("CITY_NAME", cityName);
				return true;
			}
		});
		
		//第三规则，这个针对最后一级地址可用
		rList.add(new RuleDefinition() {
			@Override
			public String getRegex() {
				return "<td>\\D*?</td>";
			}
			
			@Override
			public boolean fullData(String lineData, Map<String, String> result) {
				String cityName = lineData.substring(lineData.lastIndexOf("<td>")+4,lineData.lastIndexOf("</td>"));;
				if(cityName==null || "".equals(cityName)){
					//如果没解析到数据，证明这次是失败的！
					return false;
				}
				result.put("CITY_NAME", cityName);
				return true;
			}
		});
		return rList;
	}

}

package com.fyrj.compoment.crawler;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/***
 * 解析出城市URL
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public class CityUrlExtractHandler extends AnalyHandler{

	@Override
	List<RuleDefinition> getRuleDefinitionChain() {
		List<RuleDefinition> rList = new ArrayList<>();
		rList.add(new RuleDefinition() {
			@Override
			public String getRegex() {
				return "<a href='.*?\\.html'";
			}
			
			@Override
			public boolean fullData(String lineData, Map<String, String> result) {
				String url = lineData.substring(lineData.lastIndexOf("href='")+6,lineData.lastIndexOf(".html")+5);
				if(url==null || "".equals(url)){
					//如果没解析到数据，证明这次是失败的！
					return false;
				}
				result.put("URL",url);
				return true;
			}
		});
		return rList;
	}
}

package com.fyrj.compoment.crawler;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/***
 * 爬虫启动器
 * @author ying.cai
 * @email 919126624@qq.com
 * @version 1.0
 */
public class InvokeCenter {
		
	//页面处理器，能打开页面...
	private PageProcessor processor;
	
	//数据提取调用链，用于针对每行具体数据进行数据提取
	private AnalyHandler analyHandler;
	
	//页面过滤器，用于替换规则和页面数据
	private PageFilter pageFilter;
	
	
	public InvokeCenter( PageProcessor processor,AnalyHandler analy ) {
		this.processor = processor;
		this.analyHandler = analy;
	}
	
	public void setPageFilter(PageFilter pageFilter){
		this.pageFilter = pageFilter;
	}
	
	
	public List<Map<String,String>> start( Page page){
		List<Map<String,String>> result = new ArrayList<Map<String,String>>();
		resolverPage(page);
		result = capacityPageData(page,result,null);
		return result;
	} 
	
	public void resolverPage(Page page){
		page = processor.resolverPage(page);
	}
	
	/***
	 * 对网页进行深度解析
	 * @param page 代表一个抽象网页
	 * @param result 
	 * @param level 这个是爬虫深度！可能到了不同的深度，page匹配规则需要改变
	 * @param parentId 父数据ID
	 * @return
	 */
	public List<Map<String,String>> capacityPageData(Page page,List<Map<String,String>> result,String parentId){
		//修改数据匹配规则 ,这里采用过滤器设计
		if( null!=pageFilter ){
			pageFilter.doFilter(page);
		}
		//用正则解析页面数据
	    List<String> list = new ArrayList<String>();
	    if( page.getRegex()==null || "".equals(page.getRegex()) ){
	    	throw new RuntimeException("page regex not be null!");
	    }
	    Pattern pa = Pattern.compile(page.getRegex(), Pattern.CANON_EQ);
	    Matcher ma = pa.matcher(page.getViewData());
	    while (ma.find()){
	      list.add(ma.group());
	    }
	    
	    //接下来要对每一行数据进行分析，转换成我需要的map数据
	    for (String lineStr : list) {
	    	
	    	analyHandler.setDataStr(lineStr);
	    	//得到经过一些列解析器解析后的数据映射
	    	Map<String,String> map = analyHandler.analyStart();
	    	
	    	afterProcess(result, page.getLevel(), parentId, map);
	    	
	    	//如果存在URL，则需要继续递归
	    	if( map.containsKey("URL") ){
	    		//这行代码应该抽取出去，因为URL的构建规则随着要爬取的网页不同都是会变的！
	    		String newUrl = page.getUrl().substring(0,page.getUrl().lastIndexOf("/")+1) + map.get("URL");
	    		
	    		Page newPage = new Page();
	    		newPage.setUrl(newUrl);
	    		newPage.setRegex(page.getRegex());
	    		//继续往下走，页面深度+1
	    		newPage.setLevel(page.getLevel()+1);
	    		resolverPage(newPage);
	    		capacityPageData(newPage, result,map.get("ID"));
	    	}
	    }
	    return result;
	}

	/***
	 * 争对解析到的数据的后期处理
	 * @param result
	 * @param level
	 * @param parentId
	 * @param map
	 */
	private void afterProcess(List<Map<String, String>> result, int level, String parentId, Map<String, String> map) {
		if(map.get("CITY_NAME")!=null){
			//针对结果再做些处理！
			map.put("ID", NumberAndIdGenerator.getInstence().createId());
			map.put("NUMBER", NumberAndIdGenerator.getInstence().createNumber());
			map.put("LEVEL", level+"");
			map.put("PARENT_ID", parentId);
			result.add(map);
			System.out.println(map);
		}
	}
	
}

package com.fyrj.compoment.crawler;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
import java.util.Map;

public class Test {
	
	public static final String SQL_TEMPLATE = "insert into table_name ('id','name','number','level','parent_id')"
	+" values ('#ID#','#NAME#','#NUMBER#','#LEVEL#','#PARENT_ID#') ; ";
	
	
	public static void main(String[] args) throws IOException {
		
		//构建一个网页处理器
		PageProcessor processor = new NetworkProcessor();
		//构建两个数据提取器，专门用来提取城市地址数据
		AnalyHandler analy = new CityUrlExtractHandler();
		analy.setAnalyHandler(new CityNameExtractHandler());
		//构建一个网页数据过滤器
		PageFilter filter = new DefaultPageFilter();
		
		//构建一个网页对象
		Page page = new Page();
		page.setRegex("<td>.*?</td>");//整个网页数据提取规则
		page.setUrl("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2015/index.html");//需要爬取的url
		page.setLevel(1);	//页面深度第一次是1
		
		//构建一个爬虫启动对象
		InvokeCenter ic = new InvokeCenter(processor, analy);
		ic.setPageFilter(filter);
		
		//取得爬取结果
		List<Map<String,String>> result = ic.start(page);
		String fileDir = "C:\\Users\\Administrator\\Desktop\\cityData.txt";
		
		fullDisk(result,fileDir);
	}
	
	/***
	 * 写入磁盘
	 * @param result
	 * @throws IOException 
	 */
	public static void fullDisk(List<Map<String,String>> result,String fileDir) throws IOException{
		BufferedWriter writer = new BufferedWriter(new FileWriter(new File(fileDir)));
		try {
			for (Map<String, String> map : result) {
				writer.write( findFinalStr(map) );
				writer.newLine();
				writer.flush();
			}
		} catch (IOException e) {
			
		}finally{
			if( writer!=null ){
				writer.close();
			}
		}
	}
	
	public static String findFinalStr( Map<String, String> map ){
		String finalStr = Test.SQL_TEMPLATE.replaceAll("#ID#", map.get("ID"))
				.replaceAll("#ID#", map.get("ID"))
				.replaceAll("#NAME#", map.get("CITY_NAME"))
				.replaceAll("#NUMBER#", map.get("NUMBER"))
				.replaceAll("#LEVEL#", map.get("LEVEL"))
				.replaceAll("#PARENT_ID#", map.get("PARENT_ID")==null?"NULL":map.get("PARENT_ID"));
		return finalStr;
	}
}

写的不好的地方请见谅，这个没做太多思考写的，能满足构建全国地区数据的需求。不过你可以对InvokeCenter进行修改使其可以动态支持各种网页；