public List<InputSplit> getSplits(JobContext job) throws IOException {
StopWatch sw = new StopWatch().start();
//获取文件的最小值,如果没有配置,就按默认值1
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
//获取最大值,如果没有配置就按默认值 long的最大值
long maxSize = getMaxSplitSize(job);
// generate splits
//创建一个保存切割信息的数组
List<InputSplit> splits = new ArrayList<InputSplit>();
//将job中的所有文件信息。存在一个list中返回
List<FileStatus> files = listStatus(job);
//循环遍历所有文件
for (FileStatus file: files) {
//获取文件路径
Path path = file.getPath();
//获取文件长度
long length = file.getLen();
//当文件长度不等于0
if (length != 0) {
BlockLocation[] blkLocations;
if (file instanceof LocatedFileStatus) {
blkLocations = ((LocatedFileStatus) file).getBlockLocations();
} else {
FileSystem fs = path.getFileSystem(job.getConfiguration());
blkLocations = fs.getFileBlockLocations(file, 0, length);
}
//判断是否能切割。如果能切割那就切割,有些文件是不可以切的
if (isSplitable(job, path)) {
//获取块大小
long blockSize = file.getBlockSize();
//求切割长度。根据maxsize和minsize 和 块大小来求
// max(minsize,min(maxsize,blocksize))
//如果是按照默认配置的话。就是块大小。
long splitSize = computeSplitSize(blockSize, minSize, maxSize);
//剩余长度,一开始是lenth
long bytesRemaining = length;
//如果当前剩余长度>要切的长度的1.1倍。那么才切。否则不切了
while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
//添加切片信息
int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
splits.add(makeSplit(path, length-bytesRemaining, splitSize,
blkLocations[blkIndex].getHosts(),
blkLocations[blkIndex].getCachedHosts()));
bytesRemaining -= splitSize;
}
//最后多余了一个没有切干净,就再添加一个切片信息
if (bytesRemaining != 0) {
int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining,
blkLocations[blkIndex].getHosts(),
blkLocations[blkIndex].getCachedHosts()));
}
} //如果不能切片
else { // not splitable
//就不切。直接拿整个
splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(),
blkLocations[0].getCachedHosts()));
}
} //当文件长度等于零
else {
//Create empty hosts array for zero length files
//创建一个空的切割块添加进去
splits.add(makeSplit(path, 0, length, new String[0]));
}
}
// Save the number of input files for metrics/loadgen
job.getConfiguration().setLong(NUM_INPUT_FILES, files.size());
sw.stop();
if (LOG.isDebugEnabled()) {
LOG.debug("Total # of splits generated by getSplits: " + splits.size()
+ ", TimeTaken: " + sw.now(TimeUnit.MILLISECONDS));
}
//返回切片信息
return splits;
}