用java代码操作HDFS文件系统

工具：Maven、eclipse、部署好的hadoop
步骤：
1.下载Maven，配置环境变量
Maven下载地址
新建系统变量 MAVEN_HOME，变量值：E:\Maven\apache-maven-3.6.1
编辑系统变量 Path，添加变量值：;%MAVEN_HOME%\bin
更改本地仓库(E:\Maven\apache-maven-3.6.1\conf\settings.xml)

<localRepository>/path/to/local/repo</localRepository>
  -->
  <添加此行代码 -->
  <localRepository>D:\installed\mavenrepository</localRepository>

2.配置eclipse
Windows->preferences->Maven->Installations->add 选择Maven解压位置
Apply
Windows->preferences->Maven->User Settings->Global Settings
选择E:\Maven\apache-maven-3.6.1\conf\settings.xml Apply
3.新建并配置项目
eclipse下新建Maven Project
配置项目下pom.xml
参考配置
点击相应版本可看到对应配置

添加或更改 根据实际hadoop环境配置

 <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 	<hadoop.version>2.7.7</hadoop.version>
  </properties>

  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>3.8.1</version>
      <scope>test</scope>
    </dependency>
    <!-- jdk依赖 -->
    <dependency>
    	<groupId>jdk.tools</groupId>
    	<artifactId>jdk.tools</artifactId>
    	<version>1.8</version>
    	<scope>system</scope>
    	<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
    </dependency>
    <!-- hadoop start -->
    <dependency>
    	<groupId>org.apache.hadoop</groupId>
    	<artifactId>hadoop-common</artifactId>
    	<version>${hadoop.version}</version>
    </dependency>
    <dependency>
    	<groupId>org.apache.hadoop</groupId>
    	<artifactId>hadoop-client</artifactId>
    	<version>${hadoop.version}</version>
    </dependency>
    <dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>${hadoop.version}</version>
</dependency>
  </dependencies>

4.编写代码
新建class，编写代码

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

/**
 * 用java代码操作HDFS文件系统
 * @author
 *
 */
public class HdfsTest {
	static Configuration conf = null;
	static FileSystem fSystem = null;
	
	static {
		//有多种get方式
		conf = new Configuration();//获取配置
		//conf.set("fs.defaultFS", "hdfs://192.168.1.128:8020");//配置
		try {
			//fSystem = FileSystem.get(conf);//获取HDFS文件系统的操作对象
			fSystem = FileSystem.get(new URI("hdfs://192.168.1.128:8020"), conf, "root");//有多种get方式
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		} catch (URISyntaxException e) {
			e.printStackTrace();
		}
	}
	
	public static void main(String[] args) throws IOException, InterruptedException, URISyntaxException {
		//readFileToConcle("/word.txt");
		//readFileToLocal("/word.txt","D:\\hadoopData\\word.txt");
		copyFromLocal("D:\\hadoopData\\123.txt","/123.txt");
	}

	
	//读取HDFS文件系统中的文件到控制台
	public static void readFileToConcle(String path) throws IOException {
		//具体对文件的操作
		FSDataInputStream fis = fSystem.open(new Path(path));
		IOUtils.copyBytes(fis, System.out, 4096, true);//自动关闭
	}
	
	//读取HDFS文件系统中的文件到本地
	public static void readFileToLocal(String path,String local) throws IOException {
		OutputStream out = new FileOutputStream(new File(local));
		//具体对文件的操作
		FSDataInputStream fis = fSystem.open(new Path(path));
		IOUtils.copyBytes(fis, out, 4096, true);//自动关闭
		out.close();
	}
	
	//将本地文件上传到HDFS文件系统
	public static void copyFromLocal(String local,String path) throws IOException {
		//具体对文件的操作
		fSystem.copyFromLocalFile(new Path(local), new Path(path));
	}
}