java bytes copy_Java Text.copyBytes方法代码示例

import org.apache.hadoop.io.Text; //导入方法依赖的package包/类

private int skipUtfByteOrderMark(Text value) throws IOException {

// Strip BOM(Byte Order Mark)

// Text only support UTF-8, we only need to check UTF-8 BOM

// (0xEF,0xBB,0xBF) at the start of the text stream.

int newMaxLineLength = (int) Math.min(3L + (long) maxLineLength,

Integer.MAX_VALUE);

int newSize = in.readLine(value, newMaxLineLength, maxBytesToConsume(pos));

// Even we read 3 extra bytes for the first line,

// we won't alter existing behavior (no backwards incompat issue).

// Because the newSize is less than maxLineLength and

// the number of bytes copied to Text is always no more than newSize.

// If the return size from readLine is not less than maxLineLength,

// we will discard the current line and read the next line.

pos += newSize;

int textLength = value.getLength();

byte[] textBytes = value.getBytes();

if ((textLength >= 3) && (textBytes[0] == (byte)0xEF) &&

(textBytes[1] == (byte)0xBB) && (textBytes[2] == (byte)0xBF)) {

// find UTF-8 BOM, strip it.

LOG.info("Found UTF-8 BOM and skipped it");

textLength -= 3;

newSize -= 3;

if (textLength > 0) {

// It may work to use the same buffer and not do the copyBytes

textBytes = value.copyBytes();

value.set(textBytes, 3, textLength);

} else {

value.clear();

}

}

return newSize;

}