HDFS 文件操作

2019独角兽企业重金招聘Python工程师标准>>>

package com.dj.hadoop;


import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URI;


import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Progressable;


/**
 * @description hdfs文件操作测试
 * @author dj
 * @date 2012-10-10
 */
public class FileHandlerTest {


public static String basePath = "hdfs://192.168.10.166:54310";
public static String fileRootPath = basePath
+ "/home/hadoop/data/hadoopdata";


public static void uploadLocalFileToHDFS(File localFile, Path destPath)
throws Exception {
Configuration config = new Configuration();
FileSystem.setDefaultUri(config, new URI(basePath));
FileSystem hdfs = FileSystem.get(config);
// Path dst = new Path(fileRootPath,destPath);
// hdfs.copyFromLocalFile(src, dst);
FSDataOutputStream out = hdfs.create(destPath,new Progressable() {
@Override
public void progress() {
    System.out.println("文件进度");  
}
});
   InputStream in = new BufferedInputStream(new FileInputStream(localFile));  
IOUtils.copy(in, out);
hdfs.close();
}

/**
* 文件下载
* @param destPath
* @param localDir
* @throws Exception 
*/
public static void downloadFileFromHDFS(Path destPath,File localDir) throws Exception {
Configuration config = new Configuration();
FileSystem.setDefaultUri(config, new URI(basePath));
FileSystem hdfs = FileSystem.get(config);
if (hdfs.exists(destPath)) {
FSDataInputStream in = hdfs.open(destPath);
FileStatus stat = hdfs.getFileStatus(destPath);
byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat.getLen()))];
in.readFully(0, buffer);
in.close();
hdfs.close();

IOUtils.write(buffer, new FileOutputStream(localDir+"/"+destPath.getName()));
}
}


/**
* 删除文件
* @param destPath
* @throws Exception
*/
public static boolean deleteFile(Path destPath)throws Exception {
Configuration config = new Configuration();
FileSystem.setDefaultUri(config, new URI(basePath));
FileSystem hdfs = FileSystem.get(config);
if (hdfs.exists(destPath)) {
return hdfs.delete(destPath,true);
}
return false;
}

public static void listAll(String dir) throws Exception {
Configuration config = new Configuration();
FileSystem.setDefaultUri(config, new URI(basePath));
FileSystem fs = FileSystem.get(config);
FileStatus[] stats = fs.listStatus(new Path(basePath,dir));
for (int i = 0; stats != null && i < stats.length; ++i) {
// System.out.println(ToStringBuilder.reflectionToString(stats[i]));
if (!stats[i].isDir()) {
// regular file
System.out.println("文件:"+stats[i].getPath().toString()+"===="+stats[i].getGroup());

} else if (stats[i].isDir()) {
// dir
System.out.println("文件夹:"+stats[i].getPath().toString()+"===="+stats[i].getGroup());
}  
}
fs.close();
}


public static void createDirectory(String directoryName) throws Exception {
Configuration config = new Configuration();
FileSystem.setDefaultUri(config, new URI(basePath));
FileSystem fs = FileSystem.get(config);
System.out.println(ToStringBuilder.reflectionToString(fs));
fs.mkdirs(new Path(fileRootPath, directoryName));
fs.close();
}


public static void deleteDirectory(String directoryName) throws Exception {
Configuration config = new Configuration();
FileSystem.setDefaultUri(config, new URI(basePath));
FileSystem fs = FileSystem.get(config);
fs.delete(new Path(fileRootPath, directoryName),true);
fs.close();
}


public static void main(String[] args) throws Exception {
String directoryName = "books";
// 【1】创建文件夹
createDirectory(directoryName);
// 【2】 删除文件夹
// deleteDirectory(directoryName);
//【3】 显示文件
//  listAll("/home/hadoop/data/hadoopdata/");
//【4】 文件上传
File file = new File("G:/apiqianyi/ap2last_tmp.txt");
Path destPath = new Path(fileRootPath,directoryName+"/"+file.getName());

// uploadLocalFileToHDFS(file, destPath);
//【5】文件下载
// downloadFileFromHDFS(destPath, new File("G:/"));
//【6】删除文件
boolean flag = deleteFile(destPath);
System.out.println(flag);


}


}

转载于:https://my.oschina.net/zuiwoxing/blog/83447