hadoop 2.7 java_Java 操作 Hadoop 2.7.4

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.*;

import org.apache.hadoop.fs.permission.FsPermission;

import org.apache.hadoop.hdfs.DistributedFileSystem;

import java.io.IOException;

import java.net.URI;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.Random;

/***

@author mrchi

@Project Java_Operation_HDFS274

@File FileOperate.java

@date 2019年03月04日 16:51

@Website http://www.mrchi.cn

@version 1.0

***/

public class FileOperate {

public static DistributedFileSystem dfs = null;

public static String MASTER = "hdfs://master:9000";

public FileOperate() throws Exception {

System.out.println("初始化hadoop客户端");

// todo 设置hadoop的登录用户名决定权限问题

System.setProperty("HADOOP_USER_NAME", "root");

dfs = new DistributedFileSystem();

dfs.initialize(new URI(MASTER), new Configuration());

System.out.println("客户端连接成功");

Path workingDirectory = dfs.getWorkingDirectory();

System.out.println("工作目录：" + workingDirectory);

}

//todo 自己写的上传文件上传

public void UploadFullFile(String src, String dst) throws Exception {

String local_path;

String target_path;

if (3 > src.length() || "".equals(src)) {

local_path = "c:/Windows/logs/CBS/CBS.log";

} else {

local_path = src;

System.out.println("本地文件路径为:\t" + local_path);

}

if (3 > dst.length() || "".equals(dst)) {

target_path ="/uploadFullFile"+String.valueOf(new Random().nextInt(999))+".txt";

System.out.println("没有设置HDFS路径!默认为:\t" + target_path);

} else {

target_path = dst;

System.out.println("HDFS路径为:\t" + target_path);

}

dfs.copyFromLocalFile(new Path(local_path), new Path(target_path));

}

public void UploadFullFile() throws IOException {

String project_path = FileOperate.class.getResource("uploadFile.txt").getPath();

String local_path = project_path.substring(1, project_path.length());

String taget_path = "/uploadFullFile" + String.valueOf(new Random().nextInt(999)) + ".txt";

dfs.copyFromLocalFile(new Path(local_path), new Path(taget_path));

}

//todo 下载文件

public void DownloadFile(String hdfs,String local) throws Exception{

//使用Java API进行I/O,设置useRawLocalFileSystem=true

dfs.copyToLocalFile(false,new Path(hdfs), new Path(local),true);

System.out.println("下载完成");

}

//todo 创建文件夹

public void MkDir(String filepath) throws Exception{

boolean res = dfs.mkdirs(new Path("/"+filepath));

System.out.println("创建 "+filepath+" 目录"+(res?"创建成功":"创建失败"));

}

//todo 重命名

public void RenameFile(String oldfile,String newfile) throws Exception {

boolean res = dfs.rename(new Path("/"+oldfile), new Path("/"+newfile));

System.out.println("文件 "+(res?newfile+" 修改成功":oldfile+" 修改失败"));

}

//todo 删除目录/文件

public void DeleteDir(String deletefile) throws Exception{

boolean res = dfs.delete(new Path("/"+deletefile), false);

System.out.println("文件或目录 "+deletefile+(res?" 删除成功":" 删除失败"));

}

//todo 获取指定目录下所有文件(忽略目录)

public void FileList() throws Exception{

RemoteIterator listFiles = dfs.listFiles(new Path("/"), true);

SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");

while (listFiles.hasNext()) {

LocatedFileStatus fileStatus = listFiles.next();

// todo 权限

FsPermission permission = fileStatus.getPermission();

// todo 所属者

String owner = fileStatus.getOwner();

// todo 所属组

String group = fileStatus.getGroup();

//todo 文件大小byte

long len = fileStatus.getLen();

//todo 修改时间

long modificationTime = fileStatus.getModificationTime();

//todo HDFS 路径

Path path = fileStatus.getPath();

System.out.println("-------------------------------");

System.out.println("权限:"+permission);

System.out.println("所属者:"+owner);

System.out.println("所属组:"+group);

System.out.println("大小:"+len);

System.out.println("修改时间:"+sdf.format(new Date(modificationTime)));

System.out.println("路径:"+path);

}

}

}

pom.xml

2.7.4

org.apache.hadoop

hadoop-client

${hadoop.version}

org.apache.hadoop

hadoop-common

${hadoop.version}

org.apache.hadoop

hadoop-hdfs

${hadoop.version}