问题重现:
读取某个指定文件的某个block块的信息
import java.io.File;
import java.io.FileOutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
public class ReadSomeOneBlock {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String url = "hdfs://hadoop01:9000";
conf.set("fs.defaultFS", url);
System.setProperty("HADOOP_USER_NAME", "hadoop");
FileSystem fs = FileSystem.get(conf);
//这里直接指定了文件 如果只给路径 可以在下面循环中获取文件名做判断
RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/aa/bb/hadoop-2.7.5-centos-6.7.tar.gz"), true);
while(listFiles.hasNext()){
LocatedFileStatus lfs = listFiles.next();
//获取文件的block块
BlockLocation[] blockLocations = lfs.getBlockLocations();
//这里是读取第二个block块
if(blockLocations.length > 1){
BlockLocation bt = blockLocations[1];
//第二个block块的起始偏移量
long offset = bt.getOffset();
FSDataInputStream in = fs.open(new Path("/aa/bb/hadoop-2.7.5-centos-6.7.tar.gz"));
in.seek(offset); //设置文件的读取位置
FileOutputStream out = new FileOutputStream(new File("g:/files/hadoop_block2.tar.gz"));
IOUtils.copyBytes(in, out, 4096, true);
}
//查看文件每个block块的相关信息
/*for (BlockLocation blockLocation : blockLocations) {
System.out.println(Arrays.toString(blockLocation.getCachedHosts()));
System.out.println(Arrays.toString(blockLocation.getNames()));
System.out.println(blockLocation.getOffset());
System.out.println(Arrays.toString(blockLocation.getHosts()));
System.out.println("===================================");
}*/
}
}
}