上一篇我们讲到了用shell命令操作HDFS,现在用Java API操作HDFS。
开发工具:Idea+Maven
首先在Idea上创建一个Maven项目:hadooptrain.
POM文件如下所示:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.yoyocheknow</groupId>
<artifactId>hadoop.train</artifactId>
<version>1.0</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
</properties>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
</dependencies>
</project>
项目结构如下所示:
创建test类。
package com.yoyocheknow.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;
/**
* 类说明
* hadoop hdfs java api 操作
* @author yoyocheknow
* @date 2018/7/27 10:44
*/
public class AppTest {
FileSystem fileSystem=null;
Configuration configuration=null;
public static final String HDFS_PATH="hdfs://hadoopMaster:8020";
@Before
public void setup() throws Exception{
System.out.println("HDFS.setup");
configuration=new Configuration();
fileSystem=FileSystem.get(new URI(HDFS_PATH),configuration,"root");
}
@After
public void tearDown()throws Exception{
configuration=null;
fileSystem=null;
System.out.println("HDFS.tearDown");
}
/*
*创建目录
*/
@Test
public void mkdir()throws Exception{
fileSystem.mkdirs(new Path("/hdfsapi/test"));
}
/*
*创建文件
*/
@Test
public void create()throws Exception{
FSDataOutputStream out=fileSystem.create(new Path("/hdfsapi/test/a.txt"));
out.write("hello Hadoop".getBytes());
out.flush();
out.close();
}
/*
*读取hdfs文件并打印在控制台
*/
@Test
public void cat() throws Exception{
FSDataInputStream in=fileSystem.open(new Path("/hdfsapi/test/a.txt"));
IOUtils.copyBytes(in, System.out,1024);
in.close();
}
/*
*重命名
*/
@Test
public void rename()throws Exception{
Path oldPath= new Path("/hdfsapi/test/a.txt");
Path newPath=new Path("/hdfsapi/test/b.txt");
fileSystem.rename(oldPath,newPath);
}
/*
*从本地 上传文件到hdfs
*/
@Test
public void copyFromLocalFiles()throws Exception{
Path localParh= new Path("D:\\知乎-编辑推荐.txt");
Path newPath=new Path("/hdfsapi/test");
fileSystem.copyFromLocalFile(localParh,newPath);
}
/*
*带进度条的上传
*/
@Test
public void copyFromLocalFileswithProgress()throws Exception{
InputStream in =new BufferedInputStream(new FileInputStream(new File("D:\\[洋葱] 课时07:netty服务器的快速实现.mp4")));
FSDataOutputStream outputStream=fileSystem.create(new Path("/hdfsapi/test/netty-vedio.mp4")
, new Progressable() {
public void progress() {
System.out.print("*");
}
});
IOUtils.copyBytes(in,outputStream,4096);
}
}