以下是对HDFS的增,删, 查找操作,使用Java API实现:
package org.max.dfs;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
public class DFSFileSystemUtil {
public static void copyFile(String from, String to, FileSystem fs) throws IOException{
Path fromURI = new Path(from);
Path toURI = new Path(to);
InputStream in = null;
OutputStream out = null;
in = fs.open(fromURI);
if(fs.exists(toURI)){
fs.delete(toURI, true);
}
out = fs.create(toURI, new Progressable(){
@Override
public void progress() {
System.out.print(".");
}
});
IOUtils.copyBytes(in, out, 4096, true);
}
public static boolean deletFile(String path, FileSystem fs) throws IOException{
// return fs.deleteOnExit(new Path(path));
return fs.delete(new Path(path),true);
}
public static void displayFile(String uri, FileSystem fs) throws IOException{
FSDataInputStream in = null;
try{
in = fs.open(new Path(uri));
IOUtils.copyBytes(in, System.out, 4096, false);
// in.seek(0);
// IOUtils.copyBytes(in, System.out, 4096, false);
}finally{
IOUtils.closeStream(in);
}
}
public static void listFiles(String uri, FileSystem fs) throws IOException{
FileStatus[] status = fs.listStatus(new Path(uri));
Path[] paths = FileUtil.stat2Paths(status);
for(Path path : paths){
System.out.println(path);
}
}
public static void queryFile(String uri, String partern, String filter, FileSystem fs) throws IOException{
FileStatus[] status = fs.globStatus(new Path(partern), new RegexExcludeFilter(filter));
Path[] paths = FileUtil.stat2Paths(status);
for(Path path : paths){
System.out.println(path);
}
}
public static void main(String[] args) throws IOException {
String uri = "hdfs://localhost:9000/user/root/in/1901";
String uriNew = "hdfs://localhost:9000/user/root/in/test3.txt";
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create("hdfs://localhost:9000/user/root/in/1901"),conf);
DFSFileSystemUtil.displayFile(uri, fs);
System.out.println("********************************");
DFSFileSystemUtil.copyFile(uri, uriNew, fs);
DFSFileSystemUtil.displayFile(uriNew, fs);
DFSFileSystemUtil.listFiles("hdfs://localhost:9000/user/root/in", fs);
System.out.println(DFSFileSystemUtil.deletFile(uriNew, fs));
DFSFileSystemUtil.listFiles("hdfs://localhost:9000/user/root/in", fs);
System.out.println(DFSFileSystemUtil.deletFile(uriNew, fs));
DFSFileSystemUtil.queryFile("hdfs://localhost:9000/user/root", "in/*", "^.*/1901$", fs);
System.out.println();
DFSFileSystemUtil.queryFile("hdfs://localhost:9000/user/root", "in/*", "^.*/.*t$", fs);
}
}