package com.wisedu.hadoop.hdfs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; /** * Created by heavy_li on 2016/11/19. */ public class HdfsFileDemo { private Configuration conf ; private FileSystem fileSystem; public HdfsFileDemo(){ //加载配置文件,默认会加载CLASSPATH下的core-site.xml conf = new Configuration(); //也可以自己手动加载配置文件 如下 //conf.addResource("core-site.xml"); log(conf.get("fs.defaultFS")); try { //用超级管理员账号 不会存在权限问题 fileSystem = FileSystem.get(new URI(conf.get("fs.defaultFS")),conf,conf.get("hadoop.loginName")); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } catch (URISyntaxException e) { e.printStackTrace(); } } /** * 显示目录下的所有文件 对应ls命令 * @param fileUri */ public void showDir(String fileUri,PathFilter pathFilter){ try { FileStatus[] fileStatuses = null; if(pathFilter==null){ fileStatuses = fileSystem.listStatus(new Path(fileUri)); }else{ fileStatuses = fileSystem.listStatus(new Path(fileUri),pathFilter); } if(fileStatuses!=null&&fileStatuses.length!=0){ for(FileStatus fs : fileStatuses){ log(fs.getOwner().concat(" ").concat(fs.getGroup()).concat(" ").concat(fs.getPath().toString()).concat(" ").concat(fs.getLen()+" ")); } }else{ log("该目录下没有任何文件!"); } } catch (IOException e) { if(e instanceof FileNotFoundException){ log("目录不存在!"); }else { e.printStackTrace(); } } } /** * 读取文件,对应 cat命令 get命令 可以将字节从新生成文件 * @param fileUri */ public void readFile(String fileUri){ Path path = new Path(fileUri); try { if(fileSystem.exists(path)){ FSDataInputStream is = fileSystem.open(path); FileStatus status = fileSystem.getFileStatus(path); byte[] buffer = new byte[Integer.parseInt(String.valueOf(status.getLen()))]; is.readFully(0, buffer); is.close(); log(new String(buffer)); }else{ log("该文件不存在!"); } } catch (IOException e) { e.printStackTrace(); } } /** * 创建目录,可以创建多级目录 对应命令mkdir * @param uri */ public void mkdir(String uri){ Path path = new Path(uri); try { fileSystem.create(path); log(uri.concat(" created!")); } catch (IOException e) { e.printStackTrace(); } } /** * 删除目录或者文件 对应 rm -r 命令 * @param uri */ public void deleteDirOrFile(String uri){ Path path = new Path(uri); try { // true 相当于rm -r fileSystem.delete(path,true); log(uri.concat(" deleted!")); } catch (IOException e) { e.printStackTrace(); } } /** * 将本地文件上传到hdfs 对应命令: copyFromLocal * @param localPath * @param remoteUri 目录不存在的话会自动创建 */ public void copyFromLocal(String localPath,String remoteUri){ Path src = new Path(localPath); Path dst = new Path(remoteUri); try { fileSystem.copyFromLocalFile(src, dst); log("over!"); } catch (IOException e) { e.printStackTrace(); } } /** * 以字节形式写入hdfs系统 * @param data * @param uri */ public void writeFile(byte[] data,String uri){ Path path = new Path(uri); try { FSDataOutputStream out = fileSystem.create(path); out.write(data); log("over!"); } catch (IOException e) { e.printStackTrace(); } } /** * 直接写入字符 * @param s * @param uri */ public void writeFileStr(String s ,String uri){ Path path = new Path(uri); try { FSDataOutputStream out = fileSystem.create(path); out.writeUTF(s); log("over!"); } catch (IOException e) { e.printStackTrace(); } } /** * 查找文件块在哪个服务器上 * @param uri */ public void getFileBlockLoaction(String uri){ Path path = new Path(uri); FileStatus status = null; try { status = fileSystem.getFileStatus(path); BlockLocation[] locations = fileSystem.getFileBlockLocations(status, 0, status.getLen()); int length = locations.length; for(int i=0;i<length;i++){ String[] hosts = locations[i].getHosts(); for(String s:hosts) { log("block_" + i + "_location:" + s); } } } catch (IOException e) { e.printStackTrace(); } } /** * 查看各节点的状态 */ public void getDataNodeStatus(){ DistributedFileSystem dfs = (DistributedFileSystem)fileSystem; DatanodeInfo[] dataNodeStats = null; try { dataNodeStats = dfs.getDataNodeStats(); } catch (IOException e) { e.printStackTrace(); } if(dataNodeStats!=null) { int i=1; for (DatanodeInfo node : dataNodeStats) { log("DataNode " + i++ + " :" .concat(node.getDatanodeReport())); } } } public void close(){ if(fileSystem!=null){ try { fileSystem.close(); } catch (IOException e) { e.printStackTrace(); } } } private void log(String s){ System.out.println(s); } public static void main(String[] args){ HdfsFileDemo hdfsFileSystem = new HdfsFileDemo(); PathFilter pathFilter = new PathFilter() { public boolean accept(Path path) { if(path.getName().toLowerCase().endsWith(".txt")){ return true; }else { return false; } } }; // hdfsFileSystem.showDir("/",pathFilter); // hdfsFileSystem.readFile("/a/hello.txt"); // hdfsFileSystem.mkdir("/aa/b"); //hdfsFileSystem.deleteDirOrFile("/aa"); //hdfsFileSystem.deleteDirOrFile("/empty.txt"); // hdfsFileSystem.copyFromLocal("F:\\my picture\\Camera\\IMG_20130914_105631.jpg","/cc/dctest1.jpg"); //hdfsFileSystem.writeFileStr("I am from java! \n","/cc/a.txt"); //hdfsFileSystem.getFileBlockLoaction("/test1.txt"); hdfsFileSystem.getDataNodeStatus(); hdfsFileSystem.close(); } }
java 操作 hdfs
最新推荐文章于 2024-04-20 16:36:29 发布