java 大文件ip 统计

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Set;

public class IpStats {

	private String saveDir;
	public final static String CRLF = "\r\n";

	/**
	 * 读取IP
	 * 
	 * @param fileName
	 */
	public void readIp(String fileName) {
		try {
			BufferedReader reader = new BufferedReader(new FileReader(fileName));
			HashMap<String, String> mapListHashMap = new HashMap<String, String>();
			int  lineSize=0;
			//RandomAccessFile rndAccessFile=new RandomAccessFile(new File(fileName),"r"  );
			while (true) {
				String line = reader.readLine();
				if (line == null) {
					break;
				}


				long ipCounts = getIpCount(line);
				if (ipCounts != -1) {
					long rnd = ipCounts % 1024;
					String toFileName = rnd + ".dat";
					///System.out.println(" parpare write to file "  +toFileName );
					String strContent = mapListHashMap.get(toFileName);
					if (strContent != null) {
						strContent += line + CRLF;
						mapListHashMap.put(toFileName, strContent);
					} else {
						mapListHashMap.put(toFileName, line + CRLF);
					}
					lineSize++;

				}

				if (lineSize >= 10000) {
					Set<String> sets = mapListHashMap.keySet();
					for (String sFileName : sets) {
						write2IpFile(sFileName, mapListHashMap.get(sFileName));
						//System.out.println("write to file "  +sFileName);
					}
					System.out.println("write to file length "  + lineSize );
					lineSize=0;
					mapListHashMap.clear();
				}



			}
			reader.close();

			Set<String> sets = mapListHashMap.keySet();

			for (String sFileName : sets) {
				write2IpFile(sFileName, mapListHashMap.get(sFileName));
				System.out.println("write to file "  +sFileName);
			}
			mapListHashMap.clear();
			System.out.println("read ip complete  ");
		} catch (Exception e) {

			e.printStackTrace();
		}

	}

	/**
	 * 
	 * @param ipCountMap
	 */
	private List<ListItem> sort(HashMap<String, Long> ipCountMap) {

		Set<String> sets = ipCountMap.keySet();
		List<ListItem> ipList = new ArrayList<ListItem>();

		for (String ip : sets) {
			ListItem item = new ListItem();
			item.setIp(ip);
			item.setIpCounts(ipCountMap.get(ip));
			ipList.add(item);
		}
		System.out.println("sort ..." + ipCountMap);
		Collections.sort(ipList, new Comparator<ListItem>() {

			@Override
			public int compare(ListItem o2, ListItem o1) {
				if (o1.getIpCounts() - o2.getIpCounts() >= 1) {
					return 1;
				} else if (o1.getIpCounts() - o2.getIpCounts() == 0) {
					return 0;
				}
				return -1;
			}
		});
		return ipList;

	}

	/*
	 * read top 10 ip
	 */
	private void readTop10(File file) {
		try {
			int index = 0;
			BufferedReader reader = new BufferedReader(new FileReader(file));
			while (true) {
				String line = reader.readLine();
				if (line == null) {
					break;
				}
				if (index > 10) {
					break;
				}
				index++;
				System.out.println(line);

			}
			reader.close();

			file.renameTo(new File(saveDir+"result.txt"));
		} catch (Exception e) {

			e.printStackTrace();
		}
	}

	/*
	 * start Stat SimpleFile
	 */
	public void startWork(String dir) {
		File[] files = new File(dir).listFiles(new FileFilter() {

			@Override
			public boolean accept(File pathname) {

				return pathname.getName().endsWith(".dat");
			}
		});
		for (File file : files) {
			System.out.println("stat file ... " + file.getAbsoluteFile());
			statFile(file.getAbsolutePath());
			System.out.println("stat file ... " + file.getAbsoluteFile()+ " end ---");
		}
		File lastFile;
		while (true) {
			File[] curFiles = new File(dir).listFiles(new FileFilter() {

				@Override
				public boolean accept(File pathname) {

					return pathname.getName().endsWith(".cout");
				}
			});

			if (curFiles.length == 1) {
				lastFile = curFiles[0];
				break;
			} else {
				System.out.println("mereFile file ... "
						+ curFiles[0].getAbsolutePath());
				Merge.mereFile(saveDir, curFiles[0].getAbsolutePath(),
						curFiles[1].getAbsolutePath());

				System.out.println("mereFile end -- "
						+ curFiles[0].getAbsolutePath());
			}
		}
		System.out.println("stat file  complete ");
		readTop10(lastFile);

	}

	/*
	 * stat file
	 */
	private void statFile(String fileName) {
		HashMap<String, Long> ipCountMap = new HashMap<String, Long>();
		try {
			File curFile = new File(fileName);
			BufferedReader reader = new BufferedReader(new FileReader(curFile));
			while (true) {
				String line = reader.readLine();
				if (line == null) {
					break;
				}
				Long counts = ipCountMap.get(line);
				if (counts == null) {
					ipCountMap.put(line, Long.parseLong("1"));
				} else {
					long couts = counts.longValue() + 1;
					ipCountMap.put(line, Long.parseLong(couts + ""));
				}
			}
			reader.close();

			Set<String> sets = ipCountMap.keySet();
			String sName = new File(fileName).getName();
			int index = sName.indexOf(".");
			if (index != -1) {
				sName = sName.substring(0, index);

			}
			sName = sName + "_counts.cout";
			List<ListItem> ipList = sort(ipCountMap);
			for (ListItem ipItem : ipList) {
				write2IpFile(sName,
						ipItem.getIp() + "\t" + ipItem.getIpCounts() + CRLF);
			}
			//curFile.renameTo(new File(curFile.getAbsolutePath() + "-old"));
			 curFile.delete();
		} catch (Exception e) {

			e.printStackTrace();
		}
	}

	/*
	 * write to ip file
	 */
	private void write2IpFile(String fileName, String ipContent) {
		try {
			String filePath = saveDir + fileName;
			BufferedOutputStream fout = new BufferedOutputStream(
					new FileOutputStream(new File(filePath), true));
			fout.write(ipContent.getBytes());
			fout.close();
		} catch (Exception e) {

			e.printStackTrace();
		}

	}

	/**
	 * get ipcounts
	 * 
	 * @param ip
	 * @return
	 */
	private long getIpCount(String ip) {
		long ipcounts = 0;

		String[] ipArr = ip.split("\\.");
		if (ipArr.length != 4) {

			return -1;
		}
		ipcounts = 256 * 256 * 256 * Integer.parseInt(ipArr[0]) + 256 * 256
				* Integer.parseInt(ipArr[1]) + 256 * Integer.parseInt(ipArr[2])
				+ Integer.parseInt(ipArr[3]);
		return ipcounts;

	}

	/*
	 * 
	 * hcmfys@163.com tomjnefu 2013-07-13
	 * 
	 * @param args
	 */
	public static void main(String[] args) {
		IpStats stat = new IpStats();
		/*
		 * 测试文件夹 d:/bigfiles/ big.txt 为 2G的大文件
		 */
		String saveDir = "d:/bigfiles/";
		stat.setSaveDir(saveDir);
		stat.readIp("d:/bigfiles/big.txt");
		stat.startWork(saveDir);
	}

	public void setSaveDir(String saveDir) {
		this.saveDir = saveDir;
	}

	public String getSaveDir() {
		return saveDir;
	}

}