import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.testng.annotations.Test;
import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
/**
* java api的开发
*/
public class HdfsOperate {
/**
* 1、创建文件夹
*/
@Test
public void mkdirToHdfs() throws IOException {
//创建参数对象configuration
Configuration configuration = new Configuration();
//如果是操作hdfs上面的文件,一定要配置fs.defaultFS
configuration.set("fs.defaultFS","hdfs://node01:8020");
FileSystem fileSystem = FileSystem.get(configuration);
fileSystem.mkdirs(new Path("/fengge/dir1"));
fileSystem.close();
}
/**
* 2、文件上传
*/
@Test
public void uploadToHdfs() throws IOException {
//创建参数对象configuration
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS","hdfs://node01:8020");
//(1)获取客户端对象FileSystem
FileSystem fileSystem = FileSystem.get(configuration);
//(2)再进行文件上传
fileSystem.copyFromLocalFile(new Path("file:///E:\\hdfs\\hello.txt"),new Path("hdfs://node01:8020/fengge/dir1"));
fileSystem.close();
}
/**
* 3、文件下载
*/
@Test
public void downloadFromHdfs() throws IOException {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS","hdfs://node01:8020");
FileSystem fileSystem = FileSystem.get(configuration);
fileSystem.copyToLocalFile(new Path("hdfs://node01:8020/fengge/dir1"),new Path("file:///E:\\hdfs\\hello2.txt"));
fileSystem.close();
}
/**
* 4、删除文件
*/
@Test
public void deleteFromHdfs() throws IOException {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS","hdfs://node01:8020");
FileSystem fileSystem = FileSystem.get(configuration);
fileSystem.deleteOnExit(new Path("hdfs://node01:8020/fengge/dir1/hell.txt"));
fileSystem.close();
}
/**
* 5、文件的重命名
*/
@Test
public void renameFile() throws IOException {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS","hdfs://node01:8020");
FileSystem fileSystem = FileSystem.get(configuration);
fileSystem.rename(new Path("hdfs://node01:8020/kaikeba/dir1/hello.txt"),new Path("hdfs://node01:8020/fengge/dir1/hello3333.txt"));
fileSystem.close();
}
/**
* 6、hdfs文件相关信息
*/
@Test
public void testListFiles() throws IOException, URISyntaxException {
//(1)获取文件系统
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"),configuration);
//(2)获取文件详情
RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"),true);
while (listFiles.hasNext()){
//输出详情
LocatedFileStatus status = listFiles.next();
//获取文件名称
System.out.println(status.getPath().getName());
//获取长度
System.out.println(status.getLen());
//获取权限
System.out.println(status.getPermission());
//分组
System.out.println(status.getGroup());
//获取存储的块信息
BlockLocation[] blockLocations = status.getBlockLocations();
for(BlockLocation blockLocation : blockLocations){
//获取块存储的主机节点
String[] hosts= blockLocation.getHosts();
for(String host : hosts){
System.out.println(host);
}
}
}
fileSystem.close();
}
/**
* 7、上传
*/
@Test
public void putFileToHdfs() throws IOException, URISyntaxException {
//(1)获取文件系统
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"),configuration);
//(2)创建输入流
FileInputStream fileInputStream = new FileInputStream(new File("e:/helo.txt"));
//(3)获取输出流
FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("hdfs://node01:8020/outresult.txt"));
//(4)流对拷
IOUtils.copy(fileInputStream,fsDataOutputStream);
//(5)关闭资源
IOUtils.closeQuietly(fsDataOutputStream);
IOUtils.closeQuietly(fileInputStream);
fileSystem.close();
}
/**
* 8、从hdfs上面下载文件
*/
@Test
public void downLoadFile() throws URISyntaxException, IOException {
//(1)获取文件系统
Configuration configuration = new Configuration();
FileSystem fileSystem = FileSystem.get(URI.create("hdfs://node01:8020/outresult.txt"),configuration);
//(2)创建输入流
FSDataInputStream fsDataInputStream = fileSystem.open(new Path("hdfs://node01:8020/outresult.txt"));
//(3)创建输出流
FileOutputStream fileOutputStream = new FileOutputStream("E:\\hdfs\\aa.txt");
//(4)流对拷
IOUtils.copy(fsDataInputStream,fileOutputStream);
//(5)关闭资源
IOUtils.closeQuietly(fsDataInputStream);
IOUtils.closeQuietly(fileOutputStream);
fileSystem.close();
}
/**
* 9、小文件合并
*/
@Test
public void mergeFile() throws URISyntaxException, IOException, InterruptedException {
//获取分布式文件系统hdfs
FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"),new Configuration(),"hadoop");
FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("hdfs://node01:8020/bigfile.xml"));
LocalFileSystem localFileSystem = FileSystem.getLocal(new Configuration());
//获取本地文件系统 localFileSystem
FileStatus[] fileStatuses = localFileSystem.listStatus(new Path("E:\\hdfs\\小文件合并"));
//读取所有本地小文件,写入到hdfs的大文件里面去
for(FileStatus fileStatus : fileStatuses){
//获取每一个本地的文件路径
Path path = fileStatus.getPath();
//读取本地小文件
FSDataInputStream fsDataInputStream = localFileSystem.open(path);
IOUtils.copy(fsDataInputStream,fsDataOutputStream);
IOUtils.closeQuietly(fsDataInputStream);
}
IOUtils.closeQuietly(fsDataOutputStream);
localFileSystem.close();
fileSystem.close();
}
}
hdfs的javaAPI开发(基础)
最新推荐文章于 2022-11-06 11:17:00 发布