1.构建环境
去除编译文件share目录下所有xxxsources.jar和xxx tests.jar,将剩下的jar copy到eclispe中
2.配置hadoop环境变量
注:测试hadoop-version报错,jdk为默认路径下,需修改hadoop-env.cmd下
set JAVA_HOME=C:\PROGRA~1\Java\jdk1.8.0_201
3.代码操作hdfs
package com.zbb.hdfs;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.Test;
//hdfs客户端连接
public class HdfsClient {
//上传系统文件
public static void main(String[] args) throws IOException {
//获取客户端
Configuration conf = new Configuration();
//设置客户端连接信息
URI uri ;
FileSystem fileSystem = null ;
try {
uri = new URI("hdfs://hadoop102:9000");
fileSystem = FileSystem.get(uri, conf, "zbb");
Path path1 = new Path("e://hello.txt");
Path path2 = new Path("/hello5.txt");
fileSystem.copyFromLocalFile(path1, path2);
System.out.println("上传成功!");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally {
fileSystem.close();
}
}
@Test
public void getConfiguration() throws IOException, InterruptedException, URISyntaxException {
Configuration conf = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop102:9000"),conf,"zbb");
System.out.println(fileSystem.toString());
}
@Test
public void putFileToHdfs() throws IOException, InterruptedException, URISyntaxException {
//获取客户端
Configuration conf = new Configuration();
//conf.set("dfs.replication", "2");
//设置客户端信息
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop102:9000"), conf, "zbb");
//上传文件
fileSystem.copyFromLocalFile(new Path("e://hello.txt"), new Path("/hello2.txt"));
//关闭资源
fileSystem.close();
//总结:参数配置优先级 代码>项目classpath下自定义配置文件>客户端配置文件>默认配置
}
@Test
public void getFileFromHdfs() throws IOException, InterruptedException, URISyntaxException {
//获取客户端信息
Configuration conf = new Configuration();
//设置客户端信息
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop102:9000"), conf, "zbb");
//下载文件到本地
fileSystem.copyToLocalFile(false, new Path("/hello.txt"), new Path("e://hello1.txt"), true);
//关闭资源
fileSystem.close();
System.out.println("下载成功!");
}
@Test
public void mkdirs() throws IOException, InterruptedException, URISyntaxException {
Configuration conf = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop102:9000"), conf, "zbb");
//创建目录
fileSystem.mkdirs(new Path("/user/zbb/ouput/test"));
fileSystem.close();
}
@Test
public void del() throws IOException, InterruptedException, URISyntaxException {
Configuration conf = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop102:9000"), conf, "zbb");
fileSystem.delete(new Path("/user/zbb/ouput"), true);
fileSystem.close();
}
@Test
public void rename() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration=new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop102:9000"), configuration, "zbb");
fileSystem.rename(new Path("/hello.txt"), new Path("/hello3.txt"));
fileSystem.close();
}
@Test
public void testListFiles() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration=new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop102:9000"), configuration, "zbb");
//查看所有目录
RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"), true);
while(listFiles.hasNext()) {
LocatedFileStatus status= listFiles.next();
//文件名
String name = status.getPath().getName();
System.out.println(name);
//组名
String group = status.getGroup();
System.out.println(group);
//权限
FsPermission permission = status.getPermission();
System.out.println(permission);
//获取块的所有信息
BlockLocation[] blockLocations = status.getBlockLocations();
for (BlockLocation blockLocation : blockLocations) {
String[] hosts = blockLocation.getHosts();
for (String host : hosts) {
System.out.println(host);
}
}
System.out.println("--------------------------------------------------");
}
}
@Test
public void testListStatus() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration=new Configuration();
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop102:9000"), configuration, "zbb");
FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : listStatus) {
if(fileStatus.isFile()) {
System.out.println("f---"+fileStatus.getPath().getName());
}else {
System.out.println("d---"+fileStatus.getPath().getName());
}
}
fileSystem.close();
}
}