import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import java.io.IOException;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Random;
/***
@author mrchi
@Project Java_Operation_HDFS274
@File FileOperate.java
@date 2019年03月04日 16:51
@Website http://www.mrchi.cn
@version 1.0
***/
public class FileOperate {
public static DistributedFileSystem dfs = null;
public static String MASTER = "hdfs://master:9000";
public FileOperate() throws Exception {
System.out.println("初始化hadoop客户端");
// todo 设置hadoop的登录用户名决定权限问题
System.setProperty("HADOOP_USER_NAME", "root");
dfs = new DistributedFileSystem();
dfs.initialize(new URI(MASTER), new Configuration());
System.out.println("客户端连接成功");
Path workingDirectory = dfs.getWorkingDirectory();
System.out.println("工作目录:" + workingDirectory);
}
//todo 自己写的上传文件上传
public void UploadFullFile(String src, String dst) throws Exception {
String local_path;
String target_path;
if (3 > src.length() || "".equals(src)) {
local_path = "c:/Windows/logs/CBS/CBS.log";
} else {
local_path = src;
System.out.println("本地文件路径为:\t" + local_path);
}
if (3 > dst.length() || "".equals(dst)) {
target_path ="/uploadFullFile"+String.valueOf(new Random().nextInt(999))+".txt";
System.out.println("没有设置HDFS路径!默认为:\t" + target_path);
} else {
target_path = dst;
System.out.println("HDFS路径为:\t" + target_path);
}
dfs.copyFromLocalFile(new Path(local_path), new Path(target_path));
}
public void UploadFullFile() throws IOException {
String project_path = FileOperate.class.getResource("uploadFile.txt").getPath();
String local_path = project_path.substring(1, project_path.length());
String taget_path = "/uploadFullFile" + String.valueOf(new Random().nextInt(999)) + ".txt";
dfs.copyFromLocalFile(new Path(local_path), new Path(taget_path));
}
//todo 下载文件
public void DownloadFile(String hdfs,String local) throws Exception{
//使用Java API进行I/O,设置useRawLocalFileSystem=true
dfs.copyToLocalFile(false,new Path(hdfs), new Path(local),true);
System.out.println("下载完成");
}
//todo 创建文件夹
public void MkDir(String filepath) throws Exception{
boolean res = dfs.mkdirs(new Path("/"+filepath));
System.out.println("创建 "+filepath+" 目录"+(res?"创建成功":"创建失败"));
}
//todo 重命名
public void RenameFile(String oldfile,String newfile) throws Exception {
boolean res = dfs.rename(new Path("/"+oldfile), new Path("/"+newfile));
System.out.println("文件 "+(res?newfile+" 修改成功":oldfile+" 修改失败"));
}
//todo 删除目录/文件
public void DeleteDir(String deletefile) throws Exception{
boolean res = dfs.delete(new Path("/"+deletefile), false);
System.out.println("文件或目录 "+deletefile+(res?" 删除成功":" 删除失败"));
}
//todo 获取指定目录下所有文件(忽略目录)
public void FileList() throws Exception{
RemoteIterator listFiles = dfs.listFiles(new Path("/"), true);
SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
while (listFiles.hasNext()) {
LocatedFileStatus fileStatus = listFiles.next();
// todo 权限
FsPermission permission = fileStatus.getPermission();
// todo 所属者
String owner = fileStatus.getOwner();
// todo 所属组
String group = fileStatus.getGroup();
//todo 文件大小byte
long len = fileStatus.getLen();
//todo 修改时间
long modificationTime = fileStatus.getModificationTime();
//todo HDFS 路径
Path path = fileStatus.getPath();
System.out.println("-------------------------------");
System.out.println("权限:"+permission);
System.out.println("所属者:"+owner);
System.out.println("所属组:"+group);
System.out.println("大小:"+len);
System.out.println("修改时间:"+sdf.format(new Date(modificationTime)));
System.out.println("路径:"+path);
}
}
}
pom.xml
2.7.4
org.apache.hadoop
hadoop-client
${hadoop.version}
org.apache.hadoop
hadoop-common
${hadoop.version}
org.apache.hadoop
hadoop-hdfs
${hadoop.version}