判断两文件中的内容是否一致

1、判断两文件中的内容是否一致的shell脚本内容

#!/bin/bash

file1=$1
file2=$2

if [ -z $file1 ]; then 
exit 1
fi
if [ -z $file2 ]; then
exit 2
fi

type=$3
if [ -z $type ];then 
type=1
fi

sed -i '/^#/d' $file1
sed -i '/^#/d' $file2
sed -i '/^[ ]*$/d' $file1
sed -i '/^[ ]*$/d' $file2

java -cp /root/homework/CheckFile/IsSameFiles.jar com.utils.IsSameFiles.App $file1 $file2 $type

2、IsSameFiles.jar文件中的内容

①com.utils.IsSameFiles.App主类中的main()方法

public static void main( String[] args ) {
    	if(args.length < 2) {
    		System.out.println("You need to enter at least two parameters!");
    		return;
    	}
    	
    	String file1 = args[0];
    	String file2 = args[1];
    	int type = 1;
    	if(args.length >= 3) {
    		type = Integer.parseInt(args[2]);
    	}
    	
    	boolean result = false;
    	try {
    		// type 表示检查文件的类型，1：小文件MD5值，2：大文件MD5值，3：小文件有效内容，4：大文件有效内容
    		switch(type) {
    		case 1:{
    			// 比较两个小文件MD5是否相等
	    		long startTime = System.currentTimeMillis();
				result = CheckLittleFilesMD5.checkFiles(file1, file2);
				long endTime = System.currentTimeMillis();
				System.out.println("CheckLittleFilesMD5.checkFiles() use time " + (endTime - startTime) + "ms");
				break;
			}
    		case 2:{
    			// 比较两个大文件MD5是否相等
    			long startTime = System.currentTimeMillis();
				result = CheckBigFilesMD5.checkFiles(file1, file2);
				long endTime = System.currentTimeMillis();
				System.out.println("CheckBigFilesMD5.checkFile() use time " + (endTime - startTime) + "ms");
				break;
    		}
    		case 3:{
    			// 比较两个小文件具体内容的新增与删减
    			long startTime = System.currentTimeMillis();
				List<String> resultList = CheckLittleFilesContent.checkFiles(file1, file2);
				long endTime = System.currentTimeMillis();
				if(!resultList.isEmpty()) {
					System.out.println("+++++++++ unequal +++++++++++++");
					System.out.println("+++++++++ 标号'-'代表src文件中删除的内容，标号'+'代表opt文件中新增的内容  +++++++++++++");
					
				}
				System.out.println("CheckLittleFilesContent.checkFile() use time " + (endTime - startTime) + "ms");
				for(String res : resultList) {
					System.out.println(res);
				}
				break;
    		}
    			
    		}
    		
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
        
    }

②CheckLittleFilesContent类的具体内容

public class CheckLittleFilesContent {
	
	protected static char hexDigits[] = 
		{ '0', '1', '2', '3', '4', '5', '6','7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };  
	
	protected static MessageDigest messageDigest = null;  
    static {  
        try {  
            messageDigest = MessageDigest.getInstance("MD5");  
        } catch (NoSuchAlgorithmException nsaex) {  
            System.err.println(CheckBigFilesMD5.class.getName()+"初始化失败，MessageDigest不支持MD5!");  
            nsaex.printStackTrace();  
        }  
    } 
    
    private static final int buffSize = 1 * 1024;
	// key:文件中行内容的MD5，value:文件中行的内容
	private static Map<String, String> srcFileMap = new HashMap<String, String>();
	private static Map<String, String> optFileMap = new HashMap<String, String>();
	// List的下表代表行号，值代表文件中当前行内容的MD5
	private static List<String> srcFileMD5List = new ArrayList<String>();
	private static List<String> optFileMD5List = new ArrayList<String>();
	
	/**
     * 检查两个小文件有效内容是否相等
     * @return boolean true 相等
	 * @throws IOException 
     * @throws IOException
     */
	public static List<String> checkFiles(String srcFilePath, String optFilePath) throws IOException {
		File srcFile = new File(srcFilePath);
        File optFile = new File(optFilePath);
        
        if (srcFile.length() == optFile.length()) {
            System.out.println("+++++++++ equals +++++++++++++");
            return new ArrayList<String>();
        }
		
		// 加载需要比较的两个文件到内存中
		loadFiles(srcFilePath, optFilePath);
		
		// 求srcFileMD5List与optFileMD5List的差集
		List<String> tmplist = new ArrayList<String>();
		tmplist.addAll(srcFileMD5List);
		srcFileMD5List.removeAll(optFileMD5List);
		optFileMD5List.removeAll(tmplist);
		
//		// 求srcFileMD5List与optFileMD5List的交集
//		List<String> tmplist = new ArrayList<String>();
//		tmplist.addAll(srcFileMD5List);
//		tmplist.retainAll(optFileMD5List);
//		
//		// 求srcFileMD5List与optFileMD5List的差集
//		srcFileMD5List.removeAll(tmplist);
//		optFileMD5List.removeAll(tmplist);
		
		List<String> resultList = new ArrayList<String>();
		for(String srcFileMD5 : srcFileMD5List) {
			String srcFileContent = srcFileMap.get(srcFileMD5);
			resultList.add(srcFileContent);
		}
		for(String optFileMD5 : optFileMD5List) {
			String optFileContent = optFileMap.get(optFileMD5);
			resultList.add(optFileContent);
		}
		
		return resultList;
		
	}
	
	private static void loadFiles(String srcFilePath, String optFilePath) throws IOException {
		
		File file = new File(srcFilePath);
		InputStream in = new FileInputStream(file);
		InputStreamReader isr = new InputStreamReader(in);// InputStreamReader 是字节流通向字符流的桥梁,
		BufferedReader br = new BufferedReader(isr);
		String buff = "";
		long lineIndex = 0;
		StringBuffer sb = new StringBuffer();
		
		while((buff = br.readLine()) != null) {
			lineIndex++;
			String srcMD5 = getMD5String(buff);
			srcFileMap.put(srcMD5, sb.append("-").append(lineIndex).append("---").append(buff).toString());
			srcFileMD5List.add(srcMD5);
			sb.setLength(0);
		}
		
		file = new File(optFilePath);
		in = new FileInputStream(file);
		isr = new InputStreamReader(in);// InputStreamReader 是字节流通向字符流的桥梁,
		br = new BufferedReader(isr);
		lineIndex = 0;
		while((buff = br.readLine()) != null) {
			lineIndex++;
			String optMD5 = getMD5String(buff);
			optFileMap.put(optMD5, sb.append("+").append(lineIndex).append("---").append(buff).toString());
			optFileMD5List.add(optMD5);
			sb.setLength(0);
		}
		
		try {
			if(br != null)
				br.close();
			if(isr != null)
				isr.close();
			if(in != null) {
				in.close();
			}
		}catch(IOException e) {
			if(br != null)
				br.close();
			if(isr != null)
				isr.close();
			if(in != null) {
				in.close();
			}
		}finally {
			if(br != null)
				br=null;
			if(isr != null)
				isr=null;
			if(in != null) {
				in=null;
			}
		}
	}
	
	private static String getMD5String(String s) {  
        return getMD5String(s.getBytes());  
    }  
  
    private static String getMD5String(byte[] bytes) {  
        messageDigest.update(bytes);  
        return bufferToHex(messageDigest.digest());  
    }  
	
	private static String bufferToHex(byte bytes[]) {  
        return bufferToHex(bytes, 0, bytes.length);  
    }  
  
    private static String bufferToHex(byte bytes[], int m, int n) {  
        StringBuffer stringbuffer = new StringBuffer(2 * n);  
        int k = m + n;  
        for (int l = m; l < k; l++) {  
            appendHexPair(bytes[l], stringbuffer);  
        }  
        return stringbuffer.toString();  
    } 
    
    private static void appendHexPair(byte bt, StringBuffer stringbuffer) {  
        char c0 = hexDigits[(bt & 0xf0) >> 4];  
        char c1 = hexDigits[bt & 0xf];  
        stringbuffer.append(c0);  
        stringbuffer.append(c1);  
    }  
    

}

③CheckLittleFilesMD5类的具体内容

public class CheckLittleFilesMD5 {
	
	/**
     * 检查两个小文件MD5是否相等
     * @return boolean true 相等
     * @throws IOException
     */
    public static boolean checkFiles(String tmpFile1, String tmpFile2) throws IOException {
        File file1 = new File(tmpFile1);
        File file2 = new File(tmpFile2);
        
        if (file1.length() != file2.length()) {
            System.out.println("+++++++++ unequal +++++++++++++");
            return false;
        }
        
        InputStream fileStream1 = new FileInputStream(file1);
        InputStream fileStream2 = new FileInputStream(file2);
//        InputStream 转 byte[]
        byte[] imageByteArray = new byte[fileStream1.available()];
        return isSameFiles(imageByteArray, new byte[fileStream2.available()]);
    }
    
    /**
     * 验证两个文件字节流是否相等
     * @return boolean true 相等
     * @throws IOException
     */
    private static boolean isSameFiles(byte[] fileByte1, byte[] fileByte2) {
        String firstFileMd5 = DigestUtils.md5Hex(fileByte1);
        String secondFileMd5 = DigestUtils.md5Hex(fileByte2);
        if (firstFileMd5.equals(secondFileMd5)) {
            System.out.println("---- equals ------ md5 " + firstFileMd5);
            return true;
        } else {
            System.out.println(firstFileMd5 + " is firstFileMd5 ++ unequal ++ secondFileMd5 = " + secondFileMd5);
            return false;
        }
    }

}

④CheckBigFilesMD5类的具体内容

public class CheckBigFilesMD5 {
	
	protected static char hexDigits[] = 
		{ '0', '1', '2', '3', '4', '5', '6','7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };  
	
	protected static MessageDigest messageDigest = null;  
    static {  
        try {  
            messageDigest = MessageDigest.getInstance("MD5");  
        } catch (NoSuchAlgorithmException nsaex) {  
            System.err.println(CheckBigFilesMD5.class.getName()+"初始化失败，MessageDigest不支持MD5!");  
            nsaex.printStackTrace();  
        }  
    } 
    
    /**
     * 检查两个大文件MD5是否相等
     * @return boolean true 相等
     * @throws IOException
     */
    public static boolean checkFiles(String srcFile, String optFile) throws IOException {
		File tmpSrcFile = new File(srcFile);
		File tmpOptFile = new File(optFile);
		
		if (tmpSrcFile.length() != tmpOptFile.length()) {
            System.out.println("+++++++++ unequal +++++++++++++");
            return false;
        }
		
		String srcFileMD5 = getFileMD5String(tmpSrcFile);
		String optFileMD5 = getFileMD5String(tmpOptFile);
		
		if(srcFileMD5.equals(optFileMD5)) {
			System.out.println("---- equals ------ md5 " + srcFileMD5);
            return true;
        } else {
            System.out.println(srcFileMD5 + " is srcFileMD5 ++ unequal ++ optFileMD5 = " + optFileMD5);
            return false;
        }
    }
	
	
	public static String getFileMD5String(File file) throws IOException {  
        FileInputStream in = new FileInputStream(file);  
        FileChannel ch = in.getChannel();  
          
        //700000000 bytes are about 670M  
        int maxSize=700000000;  
          
        long startPosition=0L;  
        long step=file.length()/maxSize;  
          
        if(step == 0){  
            MappedByteBuffer byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, 0,file.length());  
            messageDigest.update(byteBuffer);  
            return bufferToHex(messageDigest.digest());  
        }  
          
        for(int i=0;i<step;i++){  
            MappedByteBuffer byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, startPosition,maxSize);  
            messageDigest.update(byteBuffer);  
            startPosition+=maxSize;  
        }  
          
        if(startPosition==file.length()){  
            return bufferToHex(messageDigest.digest());  
        }  
  
        MappedByteBuffer byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, startPosition,file.length()-startPosition);  
        messageDigest.update(byteBuffer);  
          
              
        return bufferToHex(messageDigest.digest());  
    }  
	
	public static String getMD5String(String s) {  
        return getMD5String(s.getBytes());  
    }  
  
    public static String getMD5String(byte[] bytes) {  
        messageDigest.update(bytes);  
        return bufferToHex(messageDigest.digest());  
    }  
	
	private static String bufferToHex(byte bytes[]) {  
        return bufferToHex(bytes, 0, bytes.length);  
    }  
  
    private static String bufferToHex(byte bytes[], int m, int n) {  
        StringBuffer stringbuffer = new StringBuffer(2 * n);  
        int k = m + n;  
        for (int l = m; l < k; l++) {  
            appendHexPair(bytes[l], stringbuffer);  
        }  
        return stringbuffer.toString();  
    } 
    
    private static void appendHexPair(byte bt, StringBuffer stringbuffer) {  
        char c0 = hexDigits[(bt & 0xf0) >> 4];  
        char c1 = hexDigits[bt & 0xf];  
        stringbuffer.append(c0);  
        stringbuffer.append(c1);  
    }  
    
    
    public static boolean checkPassword(String password, String md5PwdStr) {  
        String s = getMD5String(password);  
        return s.equals(md5PwdStr);  
    }  

}