JAVA API操作HDFS
注意事项
此处配置需相同
代码如下:
导入依赖
注:这里hadoop的版本最好与安装的hadoop版本一致
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.gohb</groupId>
<artifactId>hdfstest</artifactId>
<version>1.0-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.5</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
</dependencies>
</project>
log4j.properties
log4j.rootLogger=debug, stdout, R
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%5p - %m%n
log4j.appender.R=org.apache.log4j.RollingFileAppender
log4j.appender.R.File=firestorm.log
log4j.appender.R.MaxFileSize=100KB
log4j.appender.R.MaxBackupIndex=1
log4j.appender.R.layout=org.apache.log4j.PatternLayout
log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n
log4j.logger.com.codefutures=DEBUG
测试
package com.gohb;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
public class test {
/**
* 创建文件夹
* @Author gohb
*/
@Test
public void mkDir()throws Exception{
Configuration conf = new Configuration();
conf.set("fs.default.name","hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
boolean res = fs.mkdirs(new Path("/localData"));
if (res){
System.out.println("文件夹创建成功");
}else {
System.out.println("文件夹创建失败");
}
fs.close();
}
/**
* 上传文件
* @Author gohb
*/
@Test
public void uploadFile()throws Exception{
Configuration conf = new Configuration();
conf.set("fs.default.name","hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
String filename = "core-site.xml";
InputStream in = new FileInputStream("f:\\hadoop-2.7.5\\etc\\hadoop\\"+filename);
OutputStream out = fs.create(new Path("/localData/"+filename));
IOUtils.copyBytes(in,out,4096,true);
System.out.println("文件上传成功");
fs.close();
}
/**
* 下载文件
* @Author gohb
*/
@Test
public void downloadFile()throws Exception{
Configuration conf = new Configuration();
conf.set("fs.default.name","hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
String filename = "core-site.xml";
InputStream in = fs.open(new Path("/localData/"+filename));
OutputStream out = new FileOutputStream("f:\\hadoop-2.7.5\\"+filename);
IOUtils.copyBytes(in,out,4096,true);
System.out.println("文件下载成功");
fs.close();
}
/**
* 删除文件
* @Author gohb
*/
@Test
public void deleteFile()throws Exception{
Configuration conf = new Configuration();
conf.set("fs.default.name","hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
String filename = "core-site.xml";
boolean res = fs.delete(new Path("/localData/" + filename), true);
if(res){
System.out.println("文件删除成功");
}else {
System.out.println("文件删除失败");
}
fs.close();
}
/**
* 遍历文件
* @Author gohb
*/
@Test
public void listFiles()throws Exception{
Configuration conf = new Configuration();
conf.set("fs.default.name","hdfs://localhost:9000");
FileSystem fs = FileSystem.get(conf);
RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fs.listFiles(new Path("/"), true);
while (locatedFileStatusRemoteIterator.hasNext()){
System.out.println(locatedFileStatusRemoteIterator.next());
}
fs.close();
}
}