测试类:
package com._51doit.test;
import com._51doit.test.utils.FsUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Test;
import java.io.*;
import java.net.URI;
public class Test01 {
public static void main(String[] args) throws Exception {
Configuration config = new Configuration();
FileSystem fs = FileSystem.newInstance(new URI("hdfs://linux01:9000"), config,"root");
boolean flag = fs.mkdirs(new Path("/aaa/bbb"));
fs.close();if(flag){
System.out.println("创建成功");
}else{
System.out.println("创建失败");
}
}
//下载 download
@Test
public void download() throws Exception {
FileSystem fs = FsUtil.getFs();
Path path1 = new Path("/a.txt");
Path path2 = new Path("d://");
fs.copyToLocalFile(path1,path2);
fs.close();
}
//上传 upload
@Test
public void upload()throws Exception{
FileSystem fs = FsUtil.getFs();
Path localPath = new Path("d://MachineCode.xml");
Path hdfsPath = new Path("/aaa/bbb");
fs.copyFromLocalFile(localPath,hdfsPath);
fs.close();
}
/**
* 移动 rename
* 注意:目标文件夹必须要存在才能移动
* @throws Exception
*/
@Test
public void move() throws Exception{
FileSystem fs = FsUtil.getFs();
Path sourcePath = new Path("/MachineCode.xml");
Path targetPath = new Path("/bbb/a.txt");
//判断文件夹是否存在,不存在则创建
Path dir = new Path("/bbb");
if(!fs.exists(dir)){
fs.mkdirs(dir);
}
fs.rename(sourcePath,targetPath);
fs.close();
}
//删除文件夹
@Test
public void delete() throws Exception {
FileSystem fs = FsUtil.getFs();
Path path = new Path("/bbb");
fs.delete(path,true);
fs.close();
}
//读取文件 open
@Test
public void readFile() throws Exception {
FileSystem fs = FsUtil.getFs();
Path filePath = new Path("/MachineCode.xml");
FSDataInputStream fsis = fs.open(filePath);
BufferedReader br = new BufferedReader(new InputStreamReader(fsis));
String line;
while ((line=br.readLine())!=null){
System.out.println(line);
}
fsis.close();
br.close();
fs.close();
}
//追加数据
@Test
public void append() throws Exception {
FileSystem fs = FsUtil.getFs();
Path filePath = new Path("/MachineCode.xml");
FSDataOutputStream fsos = fs.append(filePath);
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fsos));
bw.write("===============追加的内容3\n");
bw.write("===============追加的内容4\n");
//fsos.close(); 此处不需要关闭,不然报错java.nio.channels.ClosedChannelException
bw.close();
fs.close();
}
//遍历目录(只查看文件,不包含文件夹)
@Test
public void catFile() throws Exception {
FileSystem fs = FsUtil.getFs();
Path path = new Path("/");
RemoteIterator<LocatedFileStatus> fileIterator = fs.listFiles(path, true);//true :递归遍历
while (fileIterator.hasNext()){
LocatedFileStatus file = fileIterator.next();
// 文件的路径
Path filePath = file.getPath();
// 从路径中获取文件名
String name = filePath.getName();
//获取文件的大小
long len = file.getLen();
short replication = file.getReplication(); // 副本数
long blockSize = file.getBlockSize(); //切块大小
// 获取数据块信息
BlockLocation[] locations = file.getBlockLocations();
for (BlockLocation blockLocation : locations) {
System.out.println("文件块信息:"+blockLocation);
// 获取数据块所在的机器
//String[] hosts = blockLocation.getHosts();
String[] names = blockLocation.getNames() ;
for (String name2 : names) {
//192.168.133.101:50010
System.out.println("文件块所在机器:"+name2);
}
}
System.out.println("文件路径:"+filePath +" 文件名:"+name+" 文件大小:"+len+" 副本个数: "+replication+" 块大小: "+blockSize);
}
fs.close();
}
//遍历目录(包括文件夹)
@Test
public void catDir() throws Exception {
FileSystem fs = FsUtil.getFs();
Path path = new Path("/");
FileStatus[] fileStatus = fs.listStatus(path);
digui(fileStatus);
fs.close();
}
private void recursionCatDir(Path path) throws Exception {
FileSystem fs = FsUtil.getFs();
FileStatus[] fileStatus = fs.listStatus(path);
digui(fileStatus);
fs.close();
}
private void digui(FileStatus[] fileStatus) throws Exception {
for (FileStatus status : fileStatus)
if (status.isDirectory()) { // 文件夹
String name = status.getPath().getName();
System.out.println("文件夹:"+name+" ,路径: "+status.getPath());
//递归读取文件夹
recursionCatDir(status.getPath());
} else { // 文件
Path filePath = status.getPath();
String name = filePath.getName();//文件名
long size = status.getLen(); //文件大小
long blockSize = status.getBlockSize(); // 切块大小
short repCount = status.getReplication();// 副本个数
System.out.println("文件名:" + name+" ,路径: "+status.getPath() + " ,文件大小:" + size + "B ,切块大小:" + blockSize + "B ,副本个数:" + repCount);
}
}
//设置配置文件方式
public void configDemo(){
// 在classPath路径下添加 hdfs-site.xml 等配置文件会自动读取
}
}
工具类:
package com._51doit.test.utils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import java.net.URI;
public class FsUtil {
public static FileSystem getFs() throws Exception {
Configuration conf = new Configuration();
return FileSystem.newInstance(new URI("hdfs://linux01:9000"),conf,"root");
}
}