关于hdfs的基本操作,
读取,上传,下载,删除:
- hdfs文件读取
package org.apache.hadoop.studyhdfs;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
public class hdfsAPI {
// 获取配置文件并得到文件系统的方法
public static FileSystem getfs() throws IOException{
// get Configreation:获取配置文件
Configuration conf =new Configuration();
//也可以通过如下来配置环境 //conf.set("fs.defaultFS", "xxx");
// getFileSystem:获取文件系统
FileSystem fs =FileSystem.get(conf);
return fs;
}
//hdfs文件读取
public static void main(String[] args) throws IOException {
// step 1:get FileSystem获取文件系统
FileSystem fs =getfs();
FSDataInputStream instream =null; //new一个输入流对象
// step 2: read path设置读取路径
Path readpath =new Path("/data/inputFiles/wc.input"); //给Path对象设定值读入路径
// step 3: open inputStream打开输入流
try{
instream = fs.open(readpath);
IOUtils.copyBytes(instream, System.out, 4096,false); //将读取文件输出
}catch(Exception e){
e.printStackTrace();
}finally{
// step 4: closeStream 关闭输入流
IOUtils.closeStream(instream);
}
}
}
- hdfs文件上传
public static void main(String[] args) throws IOException {
// step 1 :获取文件系统
FileSystem fs = getfs();
// step 2 :获取本地文件路径local File
File infile =new File("/opt/datafile/wc.input");
// step 3:获取上传路径 put File
Path outfile =new Path("/data/inputFiles/input02");
FileInputStream instream =null;
FSDataOutputStream outStream=null;
try{
// step 4 :打开输入流open inputStream
instream = new FileInputStream(infile);
// step 5:打开输出流open outputStream
outStream = fs.create(outfile);
IOUtils.copyBytes(instream, outStream, 4096, false);
System.out.print("-put Files success");
}catch(Exception e){
e.printStackTrace();
}finally{
// step 6 : 关闭流
IOUtils.closeStream(instream);
IOUtils.closeStream(outStream);
}
}
-hdfs文件下载
public static void main(String[] args) throws IOException {
// step 1 :获取文件系统
FileSystem fs =getfs();
//step 2 :获取下载文件地址,以及下载到本机的地址
fs.copyToLocalFile(new Path("/data/core.xml"), new Path("/opt/datafile/"));
//step 3 :关闭文件系统
fs.close();
System.out.print("download success");
}
`
-hdfs文件删除
// delete files
public static void main(String[] args) throws IOException {
step 1: 获取文件系统
FileSystem fs =getfs();
step 2 : 获取要删除的文件
Path path =new Path("/data/core.xml");
step 3 :对文件进行判断
if(fs.exists(path)){
fs.delete(path,true); //判断正确则删除并输出success
System.out.print("success,deleted files");
}else {
System.out.print("error delete false");//判断失败则输出error信息
}
}