pom.xml配置仓库
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
pom.xml的配置依赖
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.0</version>
</dependency>
HdfsUtil.java
package com.fengqing;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;
public class HdfsUtil {
private FileSystem fileSystem;
//构造方法
public HdfsUtil () {
init();
}
//初始化方法
private void init () {
try {
Configuration configuration = new Configuration();
//configuration.set("fs.defaultFS", "hdfs://192.168.126.130:8082");
fileSystem = FileSystem.get(new URI("hdfs://192.168.126.130:8082"), configuration, "root");
} catch (Exception e) {
e.printStackTrace();
}
}
//在HDFS上创建目录
public void mkdirs () throws Exception{
fileSystem.mkdirs(new Path("/fengqing3"));
//fileSystem.close();
}
//在HDFS上创建文件 需要关闭防火墙
public void createFile () throws Exception{
FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/fengqing/hello.txt"), new Short((short)1));
fsDataOutputStream.write("hello fengqing".getBytes());
fsDataOutputStream.flush();
fsDataOutputStream.close();
}
//查看HDFS上文件的内容
public void cat () throws Exception{
FSDataInputStream fsDataInputStream = fileSystem.open(new Path("/fengqing/hello.txt"));
IOUtils.copyBytes(fsDataInputStream, System.out, 1024);
}
//对HDFS上的文件从命名
public void rename () throws Exception{
Path srcPath = new Path("/fengqing/hello.txt");
Path dstPath = new Path("/fengqing/world.txt");
fileSystem.rename(srcPath, dstPath);
}
//上传本地文件到HDFS上
public void copyFromLocalFile () throws Exception {
Path srcPath = new Path("G:/mm.txt");
Path dstPath = new Path("/fengqing/mm.txt");
fileSystem.copyFromLocalFile(srcPath, dstPath);
}
//上传本地文件到HDFS上(带进度条)
public void copyFromLocalFileWithProgress () throws Exception {
InputStream inputStream = new BufferedInputStream(new FileInputStream("G/mm.txt"));
FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/fengqing/mm.txt"),
new Progressable() {//匿名内部类
@Override
public void progress() {
System.out.print(".");
}
});
IOUtils.copyBytes(inputStream, fsDataOutputStream, 1024);
}
//从HDFS上下载文件
public void copyToLocalFile () throws Exception{
Path srcPath = new Path("/fengqing/mm.txt");
Path dstPath = new Path("G:/mm.txt");
fileSystem.copyToLocalFile(false, srcPath, dstPath, true);
}
//列出指定目录下的文件及文件夹
public void ls () throws Exception{
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : fileStatuses) {
String type = fileStatus.isDirectory() ? "文件夹" : "文件";
FsPermission permission = fileStatus.getPermission();
String owner = fileStatus.getOwner();
Path path = fileStatus.getPath();
long len = fileStatus.getLen();
System.out.println("type:" + type);
System.out.println("permission:" + permission);
System.out.println("owner:" + owner);
System.out.println("path:" + path);
System.out.println("len:" + len);
}
}
//删除HDFS上的文件或目录
public void delete () throws Exception {
fileSystem.delete(new Path("/fengqing"), true);//true表示递归删除
}
}