java 操作 hdfs_Java代码操作HDFS

packagecom.hy.hdfs;importorg.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.*;importorg.apache.hadoop.io.IOUtils;importorg.slf4j.Logger;importorg.slf4j.LoggerFactory;importjava.io.File;importjava.io.FileOutputStream;importjava.io.IOException;importjava.io.OutputStream;importjava.net.URI;importjava.net.URISyntaxException;public classHDFSCommand {public static final Logger log = LoggerFactory.getLogger(HDFSCommand.class);public static void main(String[] args) throwsException {

String hdfsURI= "hdfs://10.1.23.240:9000";

String srcPath= "D:" + File.separator + "readme.txt";

String descPath= "/xhy";

String data= "haohaohaohaohao\r\n善字\r\n善生\r\n善行\r\n守善\r\n愿善";

Configuration conf= newConfiguration();

copyFromLocalFile(hdfsURI, srcPath, descPath, conf);

uploadFile(hdfsURI, data, descPath, conf);

RemoteIterator locatedFileStatusRemoteIterator = listFile(hdfsURI, descPath, conf, true);while(locatedFileStatusRemoteIterator.hasNext()) {

LocatedFileStatus next=locatedFileStatusRemoteIterator.next();

System.out.println("listFile：" +next.toString());

}

FileStatus[] fileStatuses=listFileAndFolder(hdfsURI, descPath, conf);for(FileStatus f : fileStatuses) {

System.out.println("listFileAndFolder：" +f.toString());

}

}/*** 本地指定路径文件上传到hdfs

*

*@paramhdfsURI

*@paramsrcPath

*@paramdescPath

*@paramconf*/

public static void copyFromLocalFile(String hdfsURI, String srcPath, String descPath, Configuration conf) throwsURISyntaxException, IOException {

log.info(">> copyFromLocalFile, srcPath is {}, descPath is {}", srcPath, descPath);

FileSystem fs= FileSystem.get(newURI(hdfsURI), conf);

fs.copyFromLocalFile(new Path(srcPath), newPath(descPath));

log.info("<< copyFromLocalFile success");

fs.close();/** 底层是通过

* fs.open(new Path(srcPath), 4096);

* fs.create(new Path(descPath));

* IOUtils.copyBytes(in, out, conf, true);*/}/*** 将数据写入到hdfs

*

*@paramhdfsURI

*@paramdata

*@paramdescPath

*@paramconf*/

public static void uploadFile(String hdfsURI, String data, String descPath, Configuration conf) throwsException {

log.info(">> uploadFile, descPath is {}, data is {}", descPath, data);

FileSystem fs= FileSystem.get(newURI(hdfsURI), conf);/*FSDataOutputStream fsOutputStream = fs.create(new Path(descPath), new Progressable() {

@Override

public void progress() {

log.info("<< 写入hdfs成功，文件路径为：{}", descPath);

}

});*/FSDataOutputStream fsOutputStream= fs.create(newPath(descPath),

()-> log.info("<< 写入hdfs成功，文件路径为：{}", descPath));

fsOutputStream.write(data.getBytes(),0, data.getBytes().length);/** 以下几种方式会出现中文乱码

* fsOutputStream.writeBytes(data);

* fsOutputStream.writeUTF(data);

* fsOutputStream.writeChars(data);*/fsOutputStream.close();

fs.close();

}/*** 查找hdfs指定路径下的文件

*

*@paramhdfsURI

*@parampath

*@paramconf

*@paramrecursive 是否递归查找

*@throwsException*/

public static RemoteIterator listFile(String hdfsURI, String path, Configuration conf, boolean recursive) throwsException {

log.info(">> listFile, path is {}, recursive is {}", path, recursive);

FileSystem fs= FileSystem.get(newURI(hdfsURI), conf);

RemoteIterator result = fs.listFiles(newPath(path), recursive);

log.info("<< listFile, result is {}", result);returnresult;

}/*** 查找hdfs指定路径下的文件和文件夹

*

*@paramhdfsURI

*@parampath

*@paramconf*/

public static FileStatus[] listFileAndFolder(String hdfsURI, String path, Configuration conf) throwsException {

log.info(">> listFileAndFolder, path is {}", path);

FileSystem fs= FileSystem.get(newURI(hdfsURI), conf);

FileStatus[] result= fs.listStatus(newPath(path));

log.info("<< listFileAndFolder, result is {}", result.toString());returnresult;//方法二

}/*** 创建文件夹

*

*@paramhdfsURI

*@parampath

*@paramconf

*@throwsException*/

public static void mkDir(String hdfsURI, String path, Configuration conf) throwsException {

log.info(">> mkDir, path is {}", path);

FileSystem fs= FileSystem.get(newURI(hdfsURI), conf);boolean result = fs.mkdirs(newPath(path));if(result) {

log.info("<< mkDir {} success", path);

}else{

log.error("<< mkDir {} error", path);

}

}/*** 删除指定路径

*

*@paramhdfsURI

*@parampath

*@paramconf

*@throwsIOException*/

public static void delete(String hdfsURI, String path, Configuration conf) throwsIOException {

log.info(">> delete, path is {}", path);

conf.set("fs.defaultFS", hdfsURI);

FileSystem fs=FileSystem.get(conf);if (!fs.exists(newPath(path))) {

log.info("<< delete {} error, path no exists", path);return;

}boolean result = fs.delete(new Path(path), true);if(result) {

log.info("<< delete {} success", path);

}else{

log.error("<< delete {} error", path);

}

}/*** 从hdfs上面下载

*

*@paramhdfsURI

*@paramsrcPath

*@paramdescPath

*@paramconf

*@throwsException*/

public static void downloadFile(String hdfsURI, String srcPath, String descPath, Configuration conf) throwsException {

log.info(">> downloadFile, srcPath is {}, descPath is {}", srcPath, descPath);

FileSystem fs= FileSystem.get(newURI(hdfsURI), conf);

FSDataInputStream in= fs.open(newPath(srcPath));

OutputStream out= new FileOutputStream(newFile(descPath));

IOUtils.copyBytes(in, out, conf);

}public static void catFile(String hdfsURI, String path, Configuration conf) throwsException {

log.info(">> catFile, path is {}", path);

FileSystem fs=FileSystem.get(URI.create(hdfsURI), conf);

FSDataInputStream in= fs.open(newPath(path));try{

IOUtils.copyBytes(in, System.out,4096, false);

}finally{

IOUtils.closeStream(in);

fs.close();

}

}

}