HDFS客户端环境准备
1.拷贝文件
2.配置HADOOP_HOME环境变量
3. 配置Path环境变量
4.创建一个Maven工程HdfsClientDemo
5.导入相应的依赖坐标+日志添加
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>RELEASE</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.8.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.8</version>
<scope>system</scope>
<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
</dependency>
</dependencies>
项目的src/main/resources目录下,新建一个文件,命名为log4j.properties
,在文件中填入
log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
6.创建包名:com.hadoop.hdfs
7.创建HdfsClient类
package com.hadoop.hdfs;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
public class HdfsClient {
public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException {
Configuration conf = new Configuration();
//conf.set("fs.defaultFS","hdfs://hadoop131:9000");
// 1 获取hdfs客户端对象
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop131:9000"),conf,"hadoop");
// 2 在hdfs上创建路径
fs.mkdirs(new Path("/131"));
// 3 关闭
fs.close();
System.out.println("over");
}
8.执行程序
HDFS的API操作
1 HDFS文件上传(测试参数优先级)
1.编写源代码
@Test
public void testCopyFromLocalFile() throws URISyntaxException, IOException, InterruptedException {
//获取hdfs对象
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop131:9000"),conf,"hadoop");
// 上传API
fs.copyFromLocalFile(new Path("d:/hi.txt"),new Path("/hi.txt"));
// 关闭
fs.close();
}
2.将hdfs-site.xml拷贝到项目的根目录下
创建
写入文件
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>
再次运行程序:
此时副本数变成了1
参数优先级排序:(1)客户端代码中设置的值 >(2)ClassPath下的用户自定义配置文件 >(3)然后是服务器的默认配置
2 HDFS文件下载
// 从hdfs下载文件
@Test
public void testCopyToLocalFile() throws URISyntaxException, IOException, InterruptedException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop131:9000"),conf,"hadoop");
//执行下载操作
fs.copyToLocalFile(new Path("/hi1.txt"),new Path("d:/hi1.txt"));
fs.close();
}
3 HDFS文件夹删除
// 删除文件
@Test
public void testDelete() throws URISyntaxException, IOException, InterruptedException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop131:9000"),conf,"hadoop");
//执行删除操作
fs.delete(new Path("/131/"),true);
fs.close();
}
131目录被删除
4 HDFS文件名更改
// 文件重命名
@Test
public void testRename() throws URISyntaxException, IOException, InterruptedException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(new