static {
System.setProperty(“hadoop.home.dir”, “E:\x3\hadoop-2.9.2”);
}
public static void main(String[] args) {
// FileSystem fs = getHadoopFileSystem();
//创建文件
/*boolean result = createPath();
System.out.print(result);*/
//上传文件1
//putFile2HDFS();
//上传文件2
// putFile2HDFS2("C:\\Users\\s1378\\Desktop\\kafka.txt","/test44");
//查询目录
// list();
//下载文件
// getFile2HDFS("/test33/kafka.txt","D:/");
//下载文件2
// getFile2HDFS2("/test33/kafka.txt","D:/test.txt");
//删除
delFileHDFS("/test1");
}
private static FileSystem getHadoopFileSystem(){
FileSystem fs = null;
//根据配置文件创建fs对象
Configuration conf = new Configuration();
//放在此处的配置,优先级最高;fs.defaultFS为必须设置的内容
conf.set("fs.defaultFS","hdfs://sunjunwei.com:9000");
try {
fs = FileSystem.get(conf);
} catch (IOException e) {
e.printStackTrace();
}
return fs;
}
//文件重命名
public static void reName(String oldName,String newName){
FileSystem fs = getFileSystem();
try {
fs.rename(new Path(oldName),new Path(newName));
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 上传文件
*/
private static void putFile2HDFS(){
FileSystem fs = getHadoopFileSystem();
Path localPath = new Path("C:\\Users\\s1378\\Desktop\\kafka.txt");
//如果路径不存在会创建;如果路径文件已存在,则覆盖
Path hdfsPath = new Path("/test33/");
//复制本地文件到HDFS
try {
fs.copyFromLocalFile(localPath,hdfsPath);
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
//上传文件2
private static void putFile2HDFS2(String localPathName,String dstPathName){
FileSystem fs = getHadoopFileSystem();
//复制本地文件到HDFS
try {
//输入流
FileInputStream in = new FileInputStream(localPathName);
//hdfs输出流
FSDataOutputStream out = fs.create(new Path(dstPathName));
IOUtils.copyBytes(in,out,4096,true);
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 创建文件夹
* @return
*/
private static boolean createPath(){
boolean result = false;
FileSystem fs = getHadoopFileSystem();
Path path = new Path("/test33");
try {
result = fs.mkdirs(path);
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return result;
}
/**
* 列出目录内容
*/
private static void list(){
FileSystem fs = getHadoopFileSystem();
Path path = new Path("/");
try {
//获取目录详细信息
FileStatus[] fss = fs.listStatus(path);
//遍历目录
for (FileStatus f:fss){
String isDir = f.isDirectory() ? "目录" : "文件";
String name = f.getPath().toString();
System.out.println(isDir+" "+name);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
fs.close();
} catch (IOException e){
e.printStackTrace();
}
}
}
//下载文件
private static void getFile2HDFS(String localPathName,String hdfsPathName){
FileSystem fs = getHadoopFileSystem();
try {
fs.copyToLocalFile(new Path(localPathName),new Path(hdfsPathName));
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
//下载文件2
private static void getFile2HDFS2(String hdfsPathName,String localPathName){
FileSystem fs = getHadoopFileSystem();
try {
FSDataInputStream open = fs.open(new Path(hdfsPathName));
FileOutputStream out = new FileOutputStream(localPathName);
IOUtils.copyBytes(open,out,4096,true);
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
//删除
private static boolean delFileHDFS(String pathName){
boolean flag = false;
FileSystem fs = getHadoopFileSystem();
try {
flag = fs.delete(new Path(pathName), true);
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
return flag;
}
}