日志文件的处理和分析

package com.tudou.thread;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.Thread.State;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class ReadProcessedFile {
	
	private static BufferedReader bReader = null;
	private static String content = null;
	private static String splitInfo[] = null;
	private static final int COUNT = 1;
	
//	private static  int test_int = 0;
	
	/*
	 * 读取文件，取出所有的INFO日志存于另一个文件中
	 */
	public static void readFile(String log,String log_info) throws IOException {
		
		File inputfile = new File(log);
		bReader = new BufferedReader(new FileReader(inputfile));
		while((content = bReader.readLine()) != null) {
			System.out.println(content);
			if(content.startsWith("2014-12-09")){
				splitInfo = content.split(" ");
				if(splitInfo[3].equals("INFO")) {
					writeFile(content,log_info);
				}
			}
		}
		bReader.close();
	}
	/*
	 * 内容写入文件
	 */
	public static void writeFile(String content,String log_info) throws IOException {
		File outputFile = new File(log_info);
		if(!outputFile.exists()) {
			outputFile.createNewFile();
		}
		
		FileWriter fWriter = new FileWriter(outputFile.getAbsoluteFile(),true);
		fWriter.write(content+"\n");
		fWriter.close();
	}
	
	/*
	 * map对象写入文件
	 */	
	public static void writeObjectToFile(Map<String,Integer> map,String final_result) throws IOException{
		File file = new File(final_result);
		if(!file.exists()){
			file.createNewFile();
		}
		FileWriter fWriter = new FileWriter(file.getAbsoluteFile(),true);
		StringBuilder param = new StringBuilder();
		for(String key : map.keySet()) {
			param.append(key+"="+map.get(key)+",");
		}
		fWriter.write(param.toString()+"\n\n");
		fWriter.close();
//		System.out.println("成功写入文件");
	}
	
	/*
	 * 将INFO日志的content部分存到另一个文件中；
	 */
	public static void readProcessedFile(String log_info,String info_content) throws IOException {
		File file = new File(log_info);
		bReader = new BufferedReader(new FileReader(file));
		while((content = bReader.readLine()) != null){
			System.out.println(content);
			splitInfo = content.split(" - ");
			if(splitInfo.length > 1){
				writeFile(splitInfo[1], info_content);
			}
		}
		bReader.close();
	}
	
	/*
	 * 统计各个s3_app类型的访问量
	 */
	public static void readLogContent(String info_content,String final_result) throws IOException {
		File file = new File(info_content);
		bReader = new BufferedReader(new FileReader(file));
		String s3_app = null;
		Map<String, Integer> map = null;
		while((content = bReader.readLine()) != null){
			if(content.trim().startsWith("s3__app=")){
				splitInfo = content.split(" ");
				s3_app = splitInfo[0].substring(9,splitInfo[0].length()-1);
				if(map == null) {
					map = new HashMap<String, Integer>();
					map.put(s3_app, COUNT) ;
				}else {
					int state = 0;
					for(String key : map.keySet()){
						state ++;
						if(s3_app.equals(key)){
							map.put(s3_app, map.get(s3_app)+1);
							break;
						}
						if(state == map.size()){
							map.put(s3_app, COUNT);
						}
					}
				}
			}
		}
		bReader.close();
		writeObjectToFile(map,final_result);
		System.out.println("Each s3_app Model的访问量->" + map);
	}
			

	
	/*
	 * s3_app访问 Each app Model的访问量
	 */
	
	public static void getVisitedInfoByLogContent(String info_content,String final_result) throws IOException{
		File file = new File(info_content);
		bReader = new BufferedReader(new FileReader(file));
		Map<String, Integer> map = null;
		String visitInfo = null;
//		String s3_app = null;
		while((content = bReader.readLine()) != null){
			if(content.trim().startsWith("s3__app=")){
				splitInfo = content.split(" ");
				visitInfo = splitInfo[0].substring(9,splitInfo[0].length()-1).concat("-"+splitInfo[5].substring(5,splitInfo[5].length()-1));
				if(map == null) {
					map = new HashMap<String ,Integer>();
					map.put(visitInfo, COUNT);
				}else {
					int state = 0;
					for(String key : map.keySet()){
						state ++;
						if(visitInfo.equals(key)){
							map.put(visitInfo, map.get(visitInfo)+1);
							break;
						}
						if(state == map.size()){
							map.put(visitInfo,COUNT);
						}
					}
				}
			}
		}
		bReader.close();
		System.out.println("s3_app each app Model的访问量->"+map);
		writeObjectToFile(map,final_result);
	}
	
	
																				
	/*
	 * each app param 的 访问量
	 */
	
	public static void getEachParamInfoByLogContent(String info_content,String final_result) throws IOException {
		
		File file = new File(info_content);
		bReader = new BufferedReader(new FileReader(file));
//		List<String> paramList = new ArrayList<String>() ;
		String param[] = null;
//		String eachParam[] = null;
		String paramArray[] = null;
		String paramInfo[] = null;
//		String splitParamInfo[] = null;
		Map<String,String> paramValue = new HashMap<String,String>();
		Map<String,Integer> statisticsParam = new HashMap<String,Integer>();
		StringBuilder totalParam = new StringBuilder();

		while((content = bReader.readLine()) != null){
//			test_int ++;
			if(content.trim().startsWith("s3__app=")){
				splitInfo = content.split(" FakeSolrParam ");
				param = splitInfo[1].split(",");
				int state = -1;
				int conLocation = 0;
				int memLocation = 0;
				for(String eachParam : param) {
//					System.out.println(eachParam);
					state ++;
					if(eachParam.trim().startsWith("[app")){
						totalParam.append(eachParam.trim()+",");
					}
					if(eachParam.trim().startsWith("sortAsc")){
						totalParam.append(eachParam.trim()+",");
						continue;
					}
					if(eachParam.trim().startsWith("sortDesc")){
						totalParam.append(eachParam.trim()+",");
						continue;
					}
					if(eachParam.trim().startsWith("conditionFields")){
//						totalParam.append(eachParam+",");
						conLocation = state;
//						System.out.println("conLocation="+conLocation);
						continue;
						
					}
					if(eachParam.trim().startsWith("memcacheTime")){
						memLocation = state;
//						System.out.println("memLocation="+memLocation);
					}
				}
				
				for(int i=conLocation;i<memLocation;i++){
					totalParam.append(param[i].trim()+"-");
				}
				paramArray = totalParam.toString().split(",");
				for(String param1:paramArray) {
					paramInfo = param1.split("=");
					paramValue.put(paramInfo[0], paramInfo[1]);
				}
//				System.out.println(test_int);
//				System.out.println(paramValue);
				if(paramValue.get("sortAsc") != null && !paramValue.get("sortAsc").trim().equals("[]")){
					String key = paramValue.get("[app")+"-"+"sortAsc"+"-"+paramValue.get("sortAsc").substring(1, paramValue.get("sortAsc").length()-1);
					if(statisticsParam.size()==0){
						statisticsParam.put(key, COUNT);
					}else {
						int state1 = 0;
						for(String statisticsKey:statisticsParam.keySet()) {
							state1 ++;
							if(key.equals(statisticsKey)){
								statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);
								break;
							}
							if(statisticsParam.size() == state1){
								statisticsParam.put(key, COUNT);
							}
						}
					}
				}
				
				if(paramValue.get("sortDesc") != null && !paramValue.get("sortDesc").trim().equals("[]")){
					String key = paramValue.get("[app")+"-"+"sortDesc"+"-"+paramValue.get("sortDesc").substring(1,paramValue.get("sortDesc").length()-1);
					if(statisticsParam.size()==0){
						statisticsParam.put(key, COUNT);
					}else {
						int state2 = 0;
						for(String statisticsKey:statisticsParam.keySet()) {
							state2 ++;
							if(key.equals(statisticsKey)){
								statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);
								break;
							}
							
							if(statisticsParam.size() == state2){
								statisticsParam.put(key, COUNT);
							}
						}
					}
				}
				
				if(paramValue.get("conditionFields") != null && !paramValue.get("conditionFields").equals("[]")){
					String conditionParam[] = paramValue.get("conditionFields").split("-");
					if(conditionParam.length != 1){
						int flag = 0;
						for(String conditionValue : conditionParam){
							flag ++;
							String key = null;
							if(flag == 1) {
								key = paramValue.get("[app")+"-"+"fq"+"-"+conditionValue.substring(1,conditionValue.length());
							}
							else if(flag == conditionParam.length){
								key = paramValue.get("[app")+"-"+"fq"+"-"+conditionValue.substring(0,conditionValue.length()-1);
							}else{
								key = paramValue.get("[app")+"-"+"fq"+"-"+conditionValue.substring(0,conditionValue.length());
							}
							
							if(!key.equalsIgnoreCase(paramValue.get("[app")+"-fq-null") && !key.equalsIgnoreCase(paramValue.get("[app")+"-fq-")){
								if(statisticsParam.size() == 0){
									statisticsParam.put(key, COUNT);
								}else {
									int state3 = 0;
									for(String statisticsKey:statisticsParam.keySet()) {
										state3 ++;
										if(key.equals(statisticsKey)){
											statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);
											break;
										}
										
										if(statisticsParam.size() == state3){
											statisticsParam.put(key, COUNT);
										}
									}
								}
							}
						}
					}else{
						String key = paramValue.get("[app")+"-"+"fq"+"-"+paramValue.get("conditionFields").substring(1,paramValue.get("conditionFields").length()-2);
						if(!key.equalsIgnoreCase(paramValue.get("[app")+"-fq-null") && !key.equalsIgnoreCase(paramValue.get("[app")+"-fq-")){
							if(statisticsParam.size()==0){
								statisticsParam.put(key, COUNT);
							}else {
								int state4 = 0;
								for(String statisticsKey:statisticsParam.keySet()) {
									state4 ++;
									if(key.equals(statisticsKey)){
										statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);
										break;
									}
									
									if(statisticsParam.size() == state4){
										statisticsParam.put(key, COUNT);
									}
								}
							}
						}
					}
				}
				paramValue.clear();
				totalParam.delete(0, totalParam.length());
			}
		}
		bReader.close();
		System.out.println("each app param 的 访问量->"+statisticsParam);
		writeObjectToFile(statisticsParam,final_result);
	}

	public static void main(String args[]) throws IOException {
		readFile("D:\\s3_interface.log.2014-12-09", "D:\\log_Info.txt");
		readProcessedFile("D:\\log_Info.txt","D:\\info_content.txt");
		readLogContent("D:\\info_content.txt","D:\\final_result.txt");
		getVisitedInfoByLogContent("D:\\info_content.txt","D:\\final_result.txt");
		getEachParamInfoByLogContent("D:\\info_content.txt","D:\\final_result.txt");
		
	}
}