java 远程 hdfs_利用JAVA API远程进行HDFS的相关操作

1 packagehdfs;2

3 import java.io.*;4 importjava.security.PrivilegedExceptionAction;5

6 importorg.apache.hadoop.conf.Configuration;7 importorg.apache.hadoop.fs.FSDataOutputStream;8 importorg.apache.hadoop.fs.FileStatus;9 importorg.apache.hadoop.fs.FileSystem;10 importorg.apache.hadoop.fs.Path;11 importorg.apache.hadoop.io.IOUtils;12 importorg.apache.hadoop.security.UserGroupInformation;13 importorg.junit.Test;14

15

16 public classOperatingFiles {17 //initialization18 //读取配置文件

19 static Configuration conf = newConfiguration();20 staticFileSystem hdfs;21

22 static{

//root是你主节点虚机的用户名23 UserGroupInformation ugi =UserGroupInformation24 .createRemoteUser("root");25 try{26 ugi.doAs(new PrivilegedExceptionAction() {27 public Void run() throwsException {28 Configuration conf = newConfiguration();

//"hdfs://lyz01:9000/"对应的是你自己的网址29 conf.set("fs.default.name", "hdfs://lyz01:9000/");30 //conf.set("hadoop.job.ugi", "root");31 //以下两行是支持 hdfs的追加 功能的：hdfs.append()

32 conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");33 conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");34 Path path = new Path("hdfs://lyz01:9000/");35 //如果在本地测试，需要使用此种方法获取文件系统

36 hdfs =FileSystem.get(path.toUri(), conf);37 //hdfs = path.getFileSystem(conf);//这个也可以38 //如果在Hadoop集群下运行，使用此种方法可以直接获取默认文件系统39 //hdfs = FileSystem.get(conf);//这个不行，这样得到的hdfs所有操作都是针对本地文件系统，而不是针对hdfs的，原因不太清楚

40 return null;41 }42 });43 } catch(IOException e) {44 //TODO Auto-generated catch block

45 e.printStackTrace();46 } catch(InterruptedException e) {47 //TODO Auto-generated catch block

48 e.printStackTrace();49 }50 }51

52 //创建hdfs目录

53 @Test54 public void createDir() throwsIOException {55 String dir = "/test2/";56 Path path = newPath(dir);57 if(hdfs.exists(path)) {58 System.out.println("dir \t" + conf.get("fs.default.name") +dir59 + "\t already exists");60 return;61 }62 hdfs.mkdirs(path);63 System.out.println("new dir \t" + conf.get("fs.default.name") +dir);64 }65

66 //文件重命名

67 @Test68 public void renameFile() throwsIOException{69 String oldName = "/reduceJoin/2.txt";70 String newName = "/reduceJoin/tb_b.txt";71 Path oldPath = newPath(oldName);72 Path newPath = newPath(newName);73 if(hdfs.exists(oldPath)){74 hdfs.rename(oldPath,newPath);75 System.out.println("rename成功！");76 }else{77 System.out.println("文件不存在!rename失败!");78 }79 }80

81 //读取文件

82 @Test83 public void readFile() throwsIOException{84 String uri = "/output2017_11_12_12_57_04/part-r-00000";85 //判断文件是否存在

86 if(!hdfs.exists(newPath(uri))){87 System.out.println("Error ; the file not exists.");88 return;89 }90 InputStream in = null;91 try{92 in = hdfs.open(newPath(uri));93 //BufferedReader bf =new BufferedReader(new InputStreamReader(in,"GB2312"));//防止中文乱码94 //复制到标准输出流

95 IOUtils.copyBytes(in, System.out, 4096,false);96 /*String line = null;97 while((line = bf.readLine()) != null){98 System.out.println(line);99 }*/

100 } catch(Exception e) {101 e.printStackTrace();102 }finally{103 IOUtils.closeStream(in);104 }105 }106

107 //从本地往HDFS上传文件

108 @Test109 public void copyFile() throwsIOException {110 String localSrc = "D:/group_max.txt";111 String hdfsDst = "/group/";112 Path src = newPath(localSrc);113 Path dst = newPath(hdfsDst);114 //本地文件不存在

115 if (!(newFile(localSrc)).exists()) {116 System.out.println("Error: local dir \t" +localSrc117 + "\t not exists.");118 return;119 }120 //hdfs路径不存在

121 if (!hdfs.exists(dst)) {122 System.out.println("Error: dest dir \t" +dst.toUri()123 + "\t not exists.");124 return;125 }126 String dstPath = dst.toUri() + "/" +src.getName();127 //System.out.println(dstPath);//"/test1/3931.jpg"128 //判断上传的文件 hdfs的目录下是否存在

129 if (hdfs.exists(newPath(dstPath))) {130 System.out.println("Warn: dest file \t" +dstPath131 + "\t already exists.");132 }else{133 //本地文件上传hdfs

134 hdfs.copyFromLocalFile(src, dst);135 //list all the files in the current direction136 //遍历文件

137 FileStatus files[] =hdfs.listStatus(dst);138 System.out.println("Upload to \t" + conf.get("fs.default.name")139 +hdfsDst);140 for(FileStatus file : files) {141 System.out.println(file.getPath());142 }143 }144 }145

146 //从HDFS 下载文件 到本地

147 @Test148 public void downloadFile() throwsIllegalArgumentException,IOException{149 String hdfsDst = "/test2/2_1";150 String localSrc = "D:/hadfs";151 Path dst = newPath(hdfsDst);152 Path src = newPath(localSrc);153 //本地的路径 + hdfs下载的文件名

154 String localFile = localSrc + "/" +dst.getName();155 //如果HDFS路径不存在

156 if(!hdfs.exists(dst.getParent())){157 System.out.println("Error : the HDFS directory:\t" + dst.getParent() + "\tdoes not exist. Please check it!");158 return;159 }160 //如果本地目录不存在，则创建

161 if(!newFile(localSrc).exists()){162 newFile(localSrc).mkdirs();163 System.out.println("Warn : The local directory does not exist. It has been automatically created for you!");164 }165 //如果本地文件存在

166 if(newFile(localFile).exists()){167 System.out.println("Error : the localSrc: \t" + localFile + "\t already exists.");168 return;169 }170 //如果HDFS文件不存在

171 if(!hdfs.exists(newPath(hdfsDst))){172 System.out.println("Error : the HDFS file: \t" + hdfsDst + "\t not exists.");173 }else{174 //HDFS下载文件到本地

175 hdfs.copyToLocalFile(false,dst,src,true);176 System.out.println("successful ：download successful! please look at: \t" +localSrc);177 }178 }179

180

181 //create a new file

182 @Test183 public voidcreateFile()184 throwsIOException {185 String fileName = "/test3/b.txt";186 String fileContent = "";187 Path dst = newPath(fileName);188 //判断 新建的文件在hdfs上是否存在

189 if(hdfs.exists(dst)){190 System.out.println("Error : the hdfs file exists.");191 }else{192 byte[] bytes =fileContent.getBytes();193 FSDataOutputStream output =hdfs.create(dst);194 output.write(bytes);195 System.out.println("new file \t" + conf.get("fs.default.name")196 +fileName);197 }198 }199

200 //追加内容到文件

201 @Test202 public voidappendFile()203 throwsIOException {204 String fileName = "/test2/file2.txt";205 String fileContent = "你好 世界";206 Path dst = newPath(fileName);207 byte[] bytes =fileContent.getBytes();208 //如果文件不存在

209 if (!hdfs.exists(dst)) {210 System.out.println("Error : the file not exists");211 return;212 }213 FSDataOutputStream output =hdfs.append(dst);214 output.write(bytes);215 System.out.println("successful: append to file \t" + conf.get("fs.default.name")216 +fileName);217 }218

219

220 //列出所有文件

221 @Test222 public void listFiles() throwsIOException {223 String dirName = "/test1";224 Path f = newPath(dirName);225 FileStatus[] status =hdfs.listStatus(f);226 System.out.println(dirName + " has all files:");227 if (status.length == 0) {228 System.out.println("nothing !");229 } else{230 for (int i = 0; i < status.length; i++) {231 System.out.println(status[i].getPath().toString());232 }233 }234 }235

236 //判断文件是否存在，存在即删除

237 @Test238 public void deleteFile() throwsIOException {239 String fileName = "/test2";240 Path f = newPath(fileName);241 boolean isExists =hdfs.exists(f);242 if (isExists) { //if exists, delete

243 boolean isDel = hdfs.delete(f, true);244 System.out.println(fileName + " delete? \t" +isDel);245 } else{246 System.out.println(fileName + " exist? \t" +notExists);247 }248 }249 }