【Hadoop】HDFS客户端操作

HDFS客户端环境准备

1．拷贝文件

2．配置HADOOP_HOME环境变量

3. 配置Path环境变量

4．创建一个Maven工程HdfsClientDemo

5．导入相应的依赖坐标+日志添加

	<dependencies>
			<dependency>
				<groupId>junit</groupId>
				<artifactId>junit</artifactId>
				<version>RELEASE</version>
			</dependency>
			<dependency>
				<groupId>org.apache.logging.log4j</groupId>
				<artifactId>log4j-core</artifactId>
		<version>2.8.2</version>
			</dependency>
			<dependency>
				<groupId>org.apache.hadoop</groupId>
				<artifactId>hadoop-common</artifactId>
				<version>2.7.2</version>
			</dependency>
			<dependency>
				<groupId>org.apache.hadoop</groupId>
				<artifactId>hadoop-client</artifactId>
				<version>2.7.2</version>
			</dependency>
			<dependency>
				<groupId>org.apache.hadoop</groupId>
				<artifactId>hadoop-hdfs</artifactId>
				<version>2.7.2</version>
			</dependency>
			<dependency>
				<groupId>jdk.tools</groupId>
				<artifactId>jdk.tools</artifactId>
				<version>1.8</version>
				<scope>system</scope>
				<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
			</dependency>
	</dependencies>

项目的src/main/resources目录下，新建一个文件，命名为log4j.properties，在文件中填入

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

6．创建包名：com.hadoop.hdfs

7．创建HdfsClient类

	package com.hadoop.hdfs;
	
	import java.io.IOException;
	import java.net.URI;
	import java.net.URISyntaxException;
	
	import org.apache.hadoop.fs.FileSystem;
	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.Path;
	
	public class HdfsClient {
   
	
	    public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException {
   
	
	        Configuration conf = new Configuration();
	        //conf.set("fs.defaultFS","hdfs://hadoop131:9000");
	
	        // 1 获取hdfs客户端对象
	        FileSystem fs = FileSystem.get(new URI("hdfs://hadoop131:9000"),conf,"hadoop");
	        // 2 在hdfs上创建路径
	        fs.mkdirs(new Path("/131"));
	        // 3 关闭
	        fs.close();
	
	        System.out.println("over");
	    }

8．执行程序

HDFS的API操作

1 HDFS文件上传（测试参数优先级）

1．编写源代码

	
	@Test
	public void testCopyFromLocalFile() throws URISyntaxException, IOException, InterruptedException {
   
	
	    //获取hdfs对象
	    Configuration conf = new Configuration();
	    FileSystem fs = FileSystem.get(new URI("hdfs://hadoop131:9000"),conf,"hadoop");
	    // 上传API
	    fs.copyFromLocalFile(new Path("d:/hi.txt"),new Path("/hi.txt"));
	    // 关闭
	    fs.close();
	}

2．将hdfs-site.xml拷贝到项目的根目录下

创建

写入文件

	        <?xml version="1.0" encoding="UTF-8"?>
	        <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
	         
	        <configuration>
	                <property>
	                        <name>dfs.replication</name>
	                <value>1</value>
	                </property>
	        </configuration>

再次运行程序：

此时副本数变成了1
参数优先级排序：（1）客户端代码中设置的值 >（2）ClassPath下的用户自定义配置文件 >（3）然后是服务器的默认配置

2 HDFS文件下载

  // 从hdfs下载文件
  @Test
  public void testCopyToLocalFile() throws URISyntaxException, IOException, InterruptedException {
   

      Configuration conf = new Configuration();
      FileSystem fs = FileSystem.get(new URI("hdfs://hadoop131:9000"),conf,"hadoop");

      //执行下载操作
      fs.copyToLocalFile(new Path("/hi1.txt"),new Path("d:/hi1.txt"));

      fs.close();

  }

3 HDFS文件夹删除

    // 删除文件
    @Test
    public void testDelete() throws URISyntaxException, IOException, InterruptedException {
   

        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://hadoop131:9000"),conf,"hadoop");

        //执行删除操作
        fs.delete(new Path("/131/"),true);

        fs.close();

    }

131目录被删除

4 HDFS文件名更改

    // 文件重命名
    @Test
    public void testRename() throws URISyntaxException, IOException, InterruptedException {
   

        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new