java IO读取UTF-8格式txt文件第一行数据隐藏的坑

一、介绍

在项目中要解析一个txt文件，文件格式如下：

1|***|***|***|***|***|***|
2|***|***|***|***|***|***|
3|***|***|***|***|***|***|
4|***|***|***|***|***|***|
5|***|***|***|***|***|***|

需要验证每行数据的开始部分是不是数字，解析代码如下：

String fieldData = IoUtil.readFileAsString(f, Charset.forName("UTF-8"));//用工具类读取文件内容
String[] strArray = fieldData.split("[\r\n]+");//用换行符将每行数据分开存入数组
for (int i = 0; i < strArray.length; i++) {
	String line = strArray[i];
	line = line.replace("[\b\r\n\t]*","");
	if (StringUtils.isEmpty(line)) {
		continue;
	}
	String[] strData = line.split("\\|");
	if (StringUtils.isNumeric(strData[0])) {//判断每一行的第一个部分是不是数字
		//省略
	}
}

当文件是UTF-8格式时，解析的第一行数据开始部分校验总是不正确，明明是数字，但StringUtils.isNumeric(strData[0])一直返回false

二、原因

在网上查阅了一些资料，发现原来是文件的开始隐藏了标识文件使用了UTF-8编码的3个字节"EF BB BF"，使用UltraEdit将文件以16进制打开也能很清楚的看到第一行开始部分的EF BB BF这三个字符。

三、解决办法

1、手动修改文件编码格式：

使用Notepad++打开txt文件执行如下操作“格式-->以UTF-8无BOM格式编码”，修改后将txt文本进行保存。

2、用代码消除：（来自网络）

        /**
	 * Generic unicode textreader, which will use BOM mark to identify the encoding
	 * to be used. If BOM is not found then use a given default or system encoding.
	 */
	public class UnicodeReader extends Reader {
		PushbackInputStream internalIn;
		InputStreamReader internalIn2 = null;
		String defaultEnc;

		private static final int BOM_SIZE = 4;

		/**
		 * 
		 * @param in
		 *            inputstream to be read
		 * @param defaultEnc
		 *            default encoding if stream does not have BOM marker. Give NULL
		 *            to use system-level default.
		 */
		UnicodeReader(InputStream in, String defaultEnc) {
			internalIn = new PushbackInputStream(in, BOM_SIZE);
			this.defaultEnc = defaultEnc;
		}

		public String getDefaultEncoding() {
			return defaultEnc;
		}

		/**
		 * Get stream encoding or NULL if stream is uninitialized. Call init() or
		 * read() method to initialize it.
		 */
		public String getEncoding() {
			if (internalIn2 == null)
				return null;
			return internalIn2.getEncoding();
		}

		/**
		 * Read-ahead four bytes and check for BOM marks. Extra bytes are unread
		 * back to the stream, only BOM bytes are skipped.
		 */
		protected void init() throws IOException {
			if (internalIn2 != null)
				return;

			String encoding;
			byte bom[] = new byte[BOM_SIZE];
			int n, unread;
			n = internalIn.read(bom, 0, bom.length);

			if ((bom[0] == (byte) 0x00) && (bom[1] == (byte) 0x00)
					&& (bom[2] == (byte) 0xFE) && (bom[3] == (byte) 0xFF)) {
				encoding = "UTF-32BE";
				unread = n - 4;
			} else if ((bom[0] == (byte) 0xFF) && (bom[1] == (byte) 0xFE)
					&& (bom[2] == (byte) 0x00) && (bom[3] == (byte) 0x00)) {
				encoding = "UTF-32LE";
				unread = n - 4;
			} else if ((bom[0] == (byte) 0xEF) && (bom[1] == (byte) 0xBB)
					&& (bom[2] == (byte) 0xBF)) {
				encoding = "UTF-8";
				unread = n - 3;
			} else if ((bom[0] == (byte) 0xFE) && (bom[1] == (byte) 0xFF)) {
				encoding = "UTF-16BE";
				unread = n - 2;
			} else if ((bom[0] == (byte) 0xFF) && (bom[1] == (byte) 0xFE)) {
				encoding = "UTF-16LE";
				unread = n - 2;
			} else {
				// Unicode BOM mark not found, unread all bytes
				encoding = defaultEnc;
				unread = n;
			}
			// System.out.println("read=" + n + ", unread=" + unread);

			if (unread > 0)
				internalIn.unread(bom, (n - unread), unread);

			// Use given encoding
			if (encoding == null) {
				internalIn2 = new InputStreamReader(internalIn);
			} else {
				internalIn2 = new InputStreamReader(internalIn, encoding);
			}
		}

		public void close() throws IOException {
			init();
			internalIn2.close();
		}

		public int read(char[] cbuf, int off, int len) throws IOException {
			init();
			return internalIn2.read(cbuf, off, len);
		}
	}

以上是转换原码，用法如下：

                File f  = new File("***.txt");
		FileInputStream in = new FileInputStream(f);
		//将FileInputStream用UnicodeReader封装即可
		BufferedReader br = new BufferedReader(new UnicodeReader(in, Charset.defaultCharset().name()));
		String line = br.readLine();
		while(line != null)
		{
			System.out.println(line);
			line = br.readLine();
		}