java操作hdfs读写

HDFS读写操作

public class HdfsOp {

public static void main(String[] args) throws Exception {

hdfsReadFile("hdfs://192.168.153.133:9000/",

"hdfs://192.168.153.133:9000/spark/sparktmp.txt" );

hdfsWriteFile("out/ww.txt","hdfs://chen133:9000/",

"hdfs://192.168.153.133:9000/spark/world.txt");

}

public static void hdfsReadFile(

String hdfsUrl,String hdfsFile) throws Exception {

Configuration cfg=new Configuration();

//"fs.defaultFS"获取配置文件中设置的hdfs端口

cfg.set("fs.defaultFS",hdfsUrl);

//获取文件

FileSystem fs=FileSystem.get(cfg);

if (!fs.exists(new Path(hdfsFile))){

throw new Exception("要读取的文件不存在！");

}

//调用ApI操作，读取文件

FSDataInputStream fis=fs.open(new Path(hdfsFile));

//文件写入到本地

FileOutputStream fos = new FileOutputStream(fileName);

byte[] buffer=new byte[2048];

int count=fis.read(buffer,0,2048);

while(count>0){//数据不足时，会继续写入补null,但在记事本中看不到

fos.write(buffer,0,2048);

count=fis.read(buffer,0,2048);

}

//IOUtils.copyBytes(fis,System.out,1024);//读取文件到控制台

fs.close();

}

public static void hdfsWriteFile(

String fileName, String hdfsUrl,String hdfsFile) throws Exception {

Configuration cfg =new Configuration();

cfg.set("fs.defaultFS",hdfsUrl);

FileSystem fs = FileSystem.get(cfg);

if (fs.exists(new Path(hdfsFile))){

throw new Exception("文件已存在！");

}

FSDataOutputStream fos = fs.create(new Path(hdfsFile));//创建文件

//将内容直接写入文件

//fos.write("你好啊，这世界".getBytes(StandardCharsets.UTF_8));

//读取本地文件上传到hdfs

FileInputStream fis=new FileInputStream(fileName);

byte[] buffer=new byte[2048];

int count=fis.read(buffer,0,2048);

while(count>0){

fos.write(buffer,0,2048);

count=fis.read(buffer,0,2048);

}

//flush()是强行将缓冲区中的内容输出，否则直到缓冲区满后才会一次性的将内容输出

//这里对这些使用不全面，稍后补充

fos.flush();

//关闭流

fs.close();

}

}