利用Java实现HDFS文件上传下载
1、pom.xml配置
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<hadoop.version>3.1.3</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>
2、创建与删除
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.IOException;
public static void main( String[] args ){
Configuration conf = new Configuration();
conf.set("fs.defaultFS","hdfs://192.168.50.102:9000");
FileSystem fs=null;
try {
fs=FileSystem.get(conf);
final String PATH="/test/kb16/hadoop/ratings.csv";
Path path = new Path(PATH);
if (fs.exists(path)) {
System.out.println("DELETE "+fs.delete(path, true));
}else{
System.out.println("CREATE "+fs.create(path));
}
} catch (IOException e) {
e.printStackTrace();
}finally {
if (fs!=null) {
try {
fs.close() ;
}catch (IOException e) {
e.printStackTrace();
}
}
}
}
3、文件上传
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.File;
import java.io.IOException;
public static void main(String[] args) {
final String formPath="E:\\ratings.csv";
if (!new File(formPath).exists()) {
System.out.println(formPath +"doesn't exits");
return;
}
Configuration conf = new Configuration();
conf.set("fs.defaultFS","hdfs://192.168.50.102:9000");
FileSystem fs=null;
try {
fs=FileSystem.get(conf);
final String toPath="/test/kb16/hive";
Path to =new Path(toPath);
if (!fs.exists(to)&& !fs.mkdirs(to)) {
System.out.println(toPath +"doesn't exit and can't be created");
return;
}
Path from=new Path(formPath);
fs.copyFromLocalFile(from, to);
System.out.println("succeed in copying from "+formPath+" to "+toPath);
} catch (IOException e) {
e.printStackTrace();
System.out.println("FAILURE");
}finally{
if (null!=fs) {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
4、文件下载
import com.google.inject.internal.cglib.core.$LocalVariablesSorter;
import com.google.inject.internal.cglib.proxy.$Factory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.File;
import java.io.IOException;
public class Download {
public static void main(String[] args) {
final String toPath = "C:\\Users\\Jialin\\Desktop";
File to = new File(toPath);
if (!to.exists()&&!to.mkdirs()) {
System.err.println(toPath + "doesn't exist and can't be created");
return;
}
Configuration config = new Configuration();
config.set("fs.defaultFS", "hdfs://192.168.50.102:9000");
FileSystem fs = null;
try {
fs = FileSystem.get(config);
final String fromPath = "/test/kb16/hive/ratings.csv";
Path from = new Path(fromPath);
if (!fs.exists(from)) {
System.err.println(toPath + "doesn't exist ");
return;
}
Path _to = new Path(toPath);
fs.copyToLocalFile(from,_to);
System.out.println("succeed in downloading from "+fromPath+" to"+toPath);
} catch (IOException e) {
e.printStackTrace();
System.out.println("FAILURE");
} finally {
if (null != fs)
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}