环境配置
Windows依赖包,拷贝hadoop-3.1.0到非中文路径(比如d:\)
https://gitee.com/eeasy/hadoop_windows_dependency/repository/archive/master.zip
配置HADOOP_HOME环境变量
配置Path环境变量 %HADOOP_HOME%\bin
检测微软运行库
用管理方式打开PowerShell,进入D:\hadoop-3.1.0\bin,执行.\winutils.exe
操作Hdfs代码
pom.xml
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.1.3</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.30</version>
</dependency>
</dependencies>
配置日志 log4j.properties
log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
编写类HdfsClient
package com.chen;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
public class HdfsClient {
private FileSystem fileSystem;
//初始化
@Before
public void init() throws URISyntaxException, IOException, InterruptedException {
//配置连接地址
URI uri = new URI("hdfs://hadoop100:8020");
//创建配置文件
Configuration configuration = new Configuration();
configuration.set("dfs.replication", "1");
String user = "root";
//获取客户端
fileSystem = FileSystem.get(uri, configuration, user);
}
//关闭连接
@After
public void close() throws IOException {
fileSystem.close();
}
//创建目录
@Test
public void mkdirs() throws IOException {
//创建文件夹
fileSystem.mkdirs(new Path("/chen"));
}
//上传文件
@Test
public void put() throws IOException {
fileSystem.copyFromLocalFile(false, true, new Path("D://chen.txt"), new Path("/chen"));
}
//下载文件
@Test
public void get() throws IOException {
//boolean useRawLocalFileSystem 是否开启文件校验
fileSystem.copyToLocalFile(false, new Path("/chen/chen.txt"), new Path("D:/chen.txt"), true);
}
//文件更名
@Test
public void rename() throws IOException {
fileSystem.rename(new Path("/chen/chen.txt"), new Path("/chen/chen1.txt"));
}
//文件移动
@Test
public void move() throws IOException {
fileSystem.rename(new Path("/chen/chen1.txt"), new Path("/chen1.txt"));
}
//删除文件和目录
@Test
public void delete() throws IOException {
fileSystem.delete(new Path("/chen"), true);
}
//文件详情查看(名称、权限、长度、块信息)
@Test
public void fileDetail() throws IOException {
RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/chen"), true);
while (listFiles.hasNext()) {
LocatedFileStatus fileStatus = listFiles.next();
System.out.println("========" + fileStatus.getPath() + "=========");
System.out.println(fileStatus.getPermission());
System.out.println(fileStatus.getOwner());
System.out.println(fileStatus.getGroup());
System.out.println(fileStatus.getLen());
System.out.println(fileStatus.getModificationTime());
System.out.println(fileStatus.getReplication());
System.out.println(fileStatus.getBlockSize());
System.out.println(fileStatus.getPath().getName());
// 获取块信息
BlockLocation[] blockLocations = fileStatus.getBlockLocations();
System.out.println(Arrays.toString(blockLocations));
}
}
//文件和文件夹判断
@Test
public void testListStatus() throws IOException {
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : fileStatuses) {
if (fileStatus.isFile()) {
System.out.println("文件:" + fileStatus.getPath().getName());
} else {
System.out.println("目录:" + fileStatus.getPath().getName());
}
}
}
}
修改参数
参数优先级排序:客户端代码中的值 > ClassPath(src/main/resources)下的用户自定义配置文件 >服务器自定义配置(安装目录/etc/hadoop/xxx-site.xml)> 服务器默认配置(安装目录/share/doc/hadoop/xxx/xxx/xxx-default.xml)
将hdfs-site.xml拷贝到项目的resources资源目录下,修改副本数
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>
修改完变为1