Java解决UTF-8的BOM问题

近日在项目中遇到一件诡异的事情，有一个txt文件，用记事本保存和notepad++ 保存，再用Java读取的时候，读出来的结果不一致。

如上面这个txt 文件，读取“自杀”2个字，虽然打印在控制台都是一样的，但是进行长度比对发现notepad++长度是2，但是记事本保存后读出来的长度竟然是3。

后面在网上查询了一下，txt文件编码格式有UTF-8和UTF-8无(BOM) 2中格式，下面来看看 这2种格式有什么不同地方。

由此可以看出，带有BOM编码格式的TXT文件多三个字节出来。而记事本保存的恰好是带有BOM编码格式的文件，而notepad++保存的编码格式确实UTF-8无(BOM)格式。

既然知道了原理，下面贴出解决方案。

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;

public class UTF8Test {

	private final static String FILE_PATH = "D:\\Workspaces\\changhong\\myUtil\\src\\main\\resources\\Sen.txt";

	
	/**
	 * 
	 * @Title readByUtf8WithBom
	 * @Description 普通方式读取 txt文件，如果用记事本保存会存在bom格式
	 * @param @return    参数
	 * @return String    返回类型
	 * @throws
	 */
	public static String readByUtf8WithBom() {
		File file = new File(FILE_PATH);

		FileInputStream in = null;
		Reader read = null;
		
		String word = null;
		
		try {
			if (file.exists() && file.isFile()) {

				in = new FileInputStream(file);

				read = new InputStreamReader(in);

				BufferedReader bf = new BufferedReader(read);

				String txt = null;
				
				while ((txt = bf.readLine()) != null) { // 读取文件

					/* 判断文本文件里面的内容是否合法 平台系统中定义 每个敏感词后加上结束标志“|1” */
					txt = txt.trim();// 去掉收尾的空格
					String flag = txt.substring(txt.lastIndexOf("|") + 1);
					if (flag.equals("1")) {

						word = txt.substring(0, txt.lastIndexOf("|"));

					}
				}
			}

		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return word;
	}
	
	/**
	 * 
	 * @Title readByUtf8WithOutBom
	 * @Description 读取 txt文件，如果存在bom格式 则去掉
	 * @param @return    参数
	 * @return String    返回类型
	 * @throws
	 */
	public static String readByUtf8WithOutBom() {
		File file = new File(FILE_PATH);

		FileInputStream in = null;
		
		String word = null;
		
		try {
			if (file.exists() && file.isFile()) {

				in = new FileInputStream(file);

				BufferedReader bf = new BufferedReader(new UnicodeReader(in,"utf-8"));

				String txt = null;
				
				while ((txt = bf.readLine()) != null) { // 读取文件

					/* 判断文本文件里面的内容是否合法 平台系统中定义 每个敏感词后加上结束标志“|1” */
					txt = txt.trim();// 去掉收尾的空格
					String flag = txt.substring(txt.lastIndexOf("|") + 1);
					if (flag.equals("1")) {

						word = txt.substring(0, txt.lastIndexOf("|"));

					}
				}
			}

		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return word;
	}
	
	
	
	public static void main(String[] args) {
		String word =UTF8Test.readByUtf8WithBom();
		
		String wordWithOutBom =UTF8Test.readByUtf8WithOutBom();
		
		System.out.println("含有bom格式的字符串:"+word+"，其长度为:"+word.length());
		System.out.println("含有bom格式的字符串:"+wordWithOutBom+"，其长度为:"+wordWithOutBom.length());
	}}

其中UnicodeReader类 是国外某个大神写的，里面具体实现感兴趣的朋友可以去研究一下。

import java.io.*;

/**
 * Generic unicode textreader, which will use BOM mark to identify the encoding
 * to be used. If BOM is not found then use a given default or system encoding.
 */
public class UnicodeReader extends Reader {
	PushbackInputStream internalIn;
	InputStreamReader internalIn2 = null;
	String defaultEnc;

	private static final int BOM_SIZE = 4;

	/**
	 * 
	 * @param in
	 *            inputstream to be read
	 * @param defaultEnc
	 *            default encoding if stream does not have BOM marker. Give NULL
	 *            to use system-level default.
	 */
	UnicodeReader(InputStream in, String defaultEnc) {
		internalIn = new PushbackInputStream(in, BOM_SIZE);
		this.defaultEnc = defaultEnc;
	}

	public String getDefaultEncoding() {
		return defaultEnc;
	}

	/**
	 * Get stream encoding or NULL if stream is uninitialized. Call init() or
	 * read() method to initialize it.
	 */
	public String getEncoding() {
		if (internalIn2 == null)
			return null;
		return internalIn2.getEncoding();
	}

	/**
	 * Read-ahead four bytes and check for BOM marks. Extra bytes are unread
	 * back to the stream, only BOM bytes are skipped.
	 */
	protected void init() throws IOException {
		if (internalIn2 != null)
			return;

		String encoding;
		byte bom[] = new byte[BOM_SIZE];
		int n, unread;
		n = internalIn.read(bom, 0, bom.length);

		if ((bom[0] == (byte) 0x00) && (bom[1] == (byte) 0x00)
				&& (bom[2] == (byte) 0xFE) && (bom[3] == (byte) 0xFF)) {
			encoding = "UTF-32BE";
			unread = n - 4;
		} else if ((bom[0] == (byte) 0xFF) && (bom[1] == (byte) 0xFE)
				&& (bom[2] == (byte) 0x00) && (bom[3] == (byte) 0x00)) {
			encoding = "UTF-32LE";
			unread = n - 4;
		} else if ((bom[0] == (byte) 0xEF) && (bom[1] == (byte) 0xBB)
				&& (bom[2] == (byte) 0xBF)) {
			encoding = "UTF-8";
			unread = n - 3;
		} else if ((bom[0] == (byte) 0xFE) && (bom[1] == (byte) 0xFF)) {
			encoding = "UTF-16BE";
			unread = n - 2;
		} else if ((bom[0] == (byte) 0xFF) && (bom[1] == (byte) 0xFE)) {
			encoding = "UTF-16LE";
			unread = n - 2;
		} else {
			// Unicode BOM mark not found, unread all bytes
			encoding = defaultEnc;
			unread = n;
		}
		// System.out.println("read=" + n + ", unread=" + unread);

		if (unread > 0)
			internalIn.unread(bom, (n - unread), unread);

		// Use given encoding
		if (encoding == null) {
			internalIn2 = new InputStreamReader(internalIn);
		} else {
			internalIn2 = new InputStreamReader(internalIn, encoding);
		}
	}

	public void close() throws IOException {
		init();
		internalIn2.close();
	}

	public int read(char[] cbuf, int off, int len) throws IOException {
		init();
		return internalIn2.read(cbuf, off, len);
	}

}

源代码demo下载