HDFS常用API

URL读取数据

InputStream in = null;
try {
	in = new URL("hdfs://hadoop:9000/input/text1.txt").openStream();
	IOUtils.copyBytes(in, System.out, 4096, false);
}finally{
	IOUtils.closeStream(in);
}

FIleSystem

读数据

String uri = "hdfs://centos1:9000/input/bank_log.txt";
FileSystem fs =FileSystem.get(URI.create(uri), conf); 
InputStream in = null;
OutputStream out = null;
try {
	in = fs.open(new Path(uri));
	IOUtils.copyBytes(in, System.out, 4096, false);
}finally{
	IOUtils.closeStream(in);
}

获取文件元数据

    Path file = new Path("/dir/file");
    FileStatus stat = fs.getFileStatus(file);
    assertThat(stat.getPath().toUri().getPath(), is("/dir/file"));
    assertThat(stat.isDirectory(), is(false));
    assertThat(stat.getLen(), is(7L));
    assertThat(stat.getModificationTime(),
        is(lessThanOrEqualTo(System.currentTimeMillis())));
    assertThat(stat.getReplication(), is((short) 1));
    assertThat(stat.getBlockSize(), is(128 * 1024 * 1024L));
    assertThat(stat.getOwner(), is(System.getProperty("user.name")));
    assertThat(stat.getGroup(), is("supergroup"));
    assertThat(stat.getPermission().toString(), is("rw-r--r--"));

列出文件

String uri = "hdfs://centos1:9000/input/";
FileSystem fs =FileSystem.get(URI.create(uri), conf); 
//FileStatus[] status = fs.globStatus(new Path("/*"), new PathFilter)
FileStatus[] status = fs.globStatus(new Path("/*"));
// FileStatus[] status =  fs.listStatus(new Path(uri));
Path[] listPath = FileUtil.stat2Paths(status);
for(Path p:listPath){
	System.out.println(p);
}

PathFilter

public class RegexExcludePathFilter implements PathFilter {
  
  private final String regex;

  public RegexExcludePathFilter(String regex) {
    this.regex = regex;
  }

  public boolean accept(Path path) {
    return !path.toString().matches(regex);
  }
}

FSDataInputStream

String uri = "hdfs://hadoop:9000/input/text1.txt";
FileSystem fs =FileSystem.get(URI.create(uri), conf); 
FSDataInputStream in = null;
try {
   in = fs.open(new Path(uri));
   IOUtils.copyBytes(in, System.out, 4096, false);
   //seek移动到文件中任意一个绝对位置
   //inputSream.skip() 只能相对当前位置定位到另一个新位置
   in.seek(0);
   IOUtils.copyBytes(in, System.out, 4096, false);
}finally{
   IOUtils.closeStream(in);
}

FSDataOutputStream

写数据

String localUri = "F:/NL/hadoop/input/bank_log.txt";
String uri = "hdfs://centos1:9000/input/bank_log.txt";
InputStream in = new BufferedInputStream(new FileInputStream(localUri));
FileSystem fs =FileSystem.get(URI.create(uri), conf); 
OutputStream out = fs.create(new Path(uri), new Progressable() {
public void progress() {
	// TODO Auto-generated method stub
	System.out.print(".");
 }
});
IOUtils.copyBytes(in, out, 4096, false);