package com.rlt.utils; import java.io.File; import java.io.IOException; import java.lang.reflect.Array; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import spire.algebra.Bool; //HDFS文件操作工具类 public class HdfsUtil { ArrayList<String> list = new ArrayList<String>(); //在hdfs系统上创建目录 public static Boolean createDir(String args) throws IOException { FileSystem fs = null; try { Configuration conf = new Configuration(); Path path = new Path(args); //设置hdfs指定用户名root,否则没有权限访问hdfs System.setProperty("HADOOP_USER_NAME", "root"); fs = path.getFileSystem(conf); fs.create(path); fs.close(); } catch (Exception e) { e.printStackTrace(); return false; }finally { fs.close(); } return true; } //在hdfs系统上删除指定目录或文件 public static Boolean deleteDir(String args) throws IOException { FileSystem fs = null; try { Configuration conf = new Configuration(); Path path = new Path(args); //设置hdfs指定用户名root,否则没有权限访问hdfs System.setProperty("HADOOP_USER_NAME", "root"); fs = path.getFileSystem(conf); fs.delete(path); } catch (Exception e) { e.printStackTrace(); return false; }finally { fs.close(); } return true; } //写文件 public static Boolean createFile (String args) throws IOException { FileSystem fs = null; FSDataOutputStream out = null; try { Configuration conf = new Configuration(); Path path = new Path(args); //设置hdfs指定用户名root,否则没有权限访问hdfs System.setProperty("HADOOP_USER_NAME", "root"); fs = path.getFileSystem(conf); out = fs.create(path); out.writeUTF("da jia hao,cai shi zhen de hao!"); } catch (Exception e) { e.printStackTrace(); return false; }finally { out.close(); fs.close(); } return true; } //读文件 public static Boolean readFile(String args) throws IOException { FileSystem fs = null; FSDataInputStream is = null; try { Configuration conf = new Configuration(); Path path = new Path(args); //设置hdfs指定用户名root,否则没有权限访问hdfs System.setProperty("HADOOP_USER_NAME", "root"); fs = path.getFileSystem(conf); if(fs.exists(path)){ is = fs.open(path); FileStatus status = fs.getFileStatus(path); byte[] buffer = new byte[Integer.parseInt(String.valueOf(status.getLen()))]; is.readFully(0, buffer); System.out.println(buffer.toString()); } } catch (Exception e) { e.printStackTrace(); return false; }finally { is.close(); fs.close(); } return true; } //上传文件到HDFS上 public static Boolean uploadFile(String srcPath,String dstPath) throws IOException { FileSystem fs = null; try { Configuration conf = new Configuration(); Path path = new Path(dstPath); //设置hdfs指定用户名root,否则没有权限访问hdfs System.setProperty("HADOOP_USER_NAME", "root"); fs = path.getFileSystem(conf); Path src = new Path(srcPath); //本地文件 Path dst = new Path(dstPath); //上传目录 //上传本地文件到hdfs上,hdfs不存在指定目录则创建该目录 fs.copyFromLocalFile(src, dst); }catch (Exception e) { e.printStackTrace(); return false; }finally { fs.close(); } return true; } //获取给定目录下的所有子目录以及子文件 public List<String> getFileList(String args)throws IOException { FileSystem fs = null; try { Configuration conf = new Configuration(); Path path = new Path(args); //设置hdfs指定用户名root,否则没有权限访问hdfs System.setProperty("HADOOP_USER_NAME", "root"); fs = path.getFileSystem(conf); this.getFile(path,fs); } catch (Exception e) { e.printStackTrace(); return null; }finally { fs.close(); } return list; } public void getFile(Path path,FileSystem fs) throws IOException { FileStatus[] fileStatus = fs.listStatus(path); for(int i=0;i<fileStatus.length;i++){ if(fileStatus[i].isDir()){ Path p = new Path(fileStatus[i].getPath().toString()); getFile(p,fs); }else{ list.add(fileStatus[i].getPath().toString()); } } } //查找某个文件在HDFS集群的位置 public static void getFilePath(String args)throws IOException { FileSystem fs = null; try { Configuration conf = new Configuration(); Path path = new Path(args); //设置hdfs指定用户名root,否则没有权限访问hdfs System.setProperty("HADOOP_USER_NAME", "root"); fs = path.getFileSystem(conf); FileStatus status = fs.getFileStatus(path); BlockLocation[] locations = fs.getFileBlockLocations(status, 0, status.getLen()); int length = locations.length; for(int i=0;i<length;i++){ String[] hosts = locations[i].getHosts(); System.out.println("block_" + i + "_location:" + hosts[i]); } } catch (Exception e) { e.printStackTrace(); }finally { fs.close(); } } //HDFS集群上所有节点名称信息 public static void getHDFSNode(String args) throws IOException{ FileSystem fs = null; DistributedFileSystem dfs = null; try { Configuration conf = new Configuration(); Path path = new Path(args); //设置hdfs指定用户名root,否则没有权限访问hdfs System.setProperty("HADOOP_USER_NAME", "root"); fs = path.getFileSystem(conf); dfs = (DistributedFileSystem)fs; DatanodeInfo[] dataNodeStats = dfs.getDataNodeStats(); for(int i=0;i<dataNodeStats.length;i++){ System.out.println("DataNode_" + i + "_Node:" + dataNodeStats[i].getHostName()); } } catch (Exception e) { e.printStackTrace(); }finally { dfs.close(); fs.close(); } } public static void main(String[] args) throws IOException { // createDir("hdfs://master:9000/dict"); // deleteDir("hdfs://master:9000/test.txt"); // createFile("hdfs://master:9000/test.txt"); // readFile("hdfs://master:9000/test.txt"); // uploadFile("C:/Users/zixiang/Desktop/sql.txt","hdfs://master:9000/test"); // HdfsUtil hdfsUtil = new HdfsUtil(); // ArrayList<String> list = (ArrayList<String>) hdfsUtil.getFileList("hdfs://master:9000/sougoudict"); // for(String i:list){ // System.out.println(i); // } getFilePath("hdfs://master:9000/test"); getHDFSNode("hdfs://master:9000"); } }
java编写HDFS操作工具类
最新推荐文章于 2023-09-08 11:04:09 发布