工具:Maven、eclipse、部署好的hadoop
步骤:
1.下载Maven,配置环境变量
Maven下载地址
新建系统变量 MAVEN_HOME,变量值:E:\Maven\apache-maven-3.6.1
编辑系统变量 Path,添加变量值:;%MAVEN_HOME%\bin
更改本地仓库(E:\Maven\apache-maven-3.6.1\conf\settings.xml)
<localRepository>/path/to/local/repo</localRepository>
-->
<添加此行代码 -->
<localRepository>D:\installed\mavenrepository</localRepository>
2.配置eclipse
Windows->preferences->Maven->Installations->add 选择Maven解压位置
Apply
Windows->preferences->Maven->User Settings->Global Settings
选择E:\Maven\apache-maven-3.6.1\conf\settings.xml Apply
3.新建并配置项目
eclipse下新建Maven Project
配置项目下pom.xml
参考配置
点击相应版本可看到对应配置
添加或更改 根据实际hadoop环境配置
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>2.7.7</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<!-- jdk依赖 -->
<dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.8</version>
<scope>system</scope>
<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
</dependency>
<!-- hadoop start -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>
4.编写代码
新建class,编写代码
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
/**
* 用java代码操作HDFS文件系统
* @author
*
*/
public class HdfsTest {
static Configuration conf = null;
static FileSystem fSystem = null;
static {
//有多种get方式
conf = new Configuration();//获取配置
//conf.set("fs.defaultFS", "hdfs://192.168.1.128:8020");//配置
try {
//fSystem = FileSystem.get(conf);//获取HDFS文件系统的操作对象
fSystem = FileSystem.get(new URI("hdfs://192.168.1.128:8020"), conf, "root");//有多种get方式
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException, InterruptedException, URISyntaxException {
//readFileToConcle("/word.txt");
//readFileToLocal("/word.txt","D:\\hadoopData\\word.txt");
copyFromLocal("D:\\hadoopData\\123.txt","/123.txt");
}
//读取HDFS文件系统中的文件到控制台
public static void readFileToConcle(String path) throws IOException {
//具体对文件的操作
FSDataInputStream fis = fSystem.open(new Path(path));
IOUtils.copyBytes(fis, System.out, 4096, true);//自动关闭
}
//读取HDFS文件系统中的文件到本地
public static void readFileToLocal(String path,String local) throws IOException {
OutputStream out = new FileOutputStream(new File(local));
//具体对文件的操作
FSDataInputStream fis = fSystem.open(new Path(path));
IOUtils.copyBytes(fis, out, 4096, true);//自动关闭
out.close();
}
//将本地文件上传到HDFS文件系统
public static void copyFromLocal(String local,String path) throws IOException {
//具体对文件的操作
fSystem.copyFromLocalFile(new Path(local), new Path(path));
}
}