Hadoop分布式文件系统HDFS的JAVA-API实战

Hadoop分布式文件系统HDFS的实战

HDFS JAVA API

搭建Linux下Eclipse开发环境

• 将eclipse-jee-juno-SR2-linux-gtk-x86_64.tar.gz和hadoop-eclipse-plugin-2.7.1.jar上传/usr/local目录下解压

[root@hadoop0 local]# tar -xvf eclipse-jee-juno-SR2-linux-gtk-x86_64.tar.gz 

为Eclipse安装Hadoop插件

图片15

---

• 打开Eclipse，直接选择/root/workpace，单击菜单中的Window->Open Perspective->Other->Map/Reduce

图片16

---

• 在Map/Reduce Location视图里单击右键，new Hadoop location…

图片17

---

• 填入Hadoop基本信息

图片18

---

• 完成第一大步，文件目录终于可以可视化

图片19

---

• 新建一个工程Project->new Project

图片20

---

• 选择Java，取名hdsf，yes。生成了一个hdfs工程

图片21

---

jar包导入

• 将已经准备好的3hdfs工程复制到eclipse中,因为篇幅的关系，代码放在最后

图片22

---

• 这么多爆红是因为没有jar包，在/usr/local/hadoop/share/hadoop中四个jar包

图片23

---

• 新建alllib放所有需要的jar包,common，hdfs，mapreduce，yarn四个子目录下所有jar包，包括lib中的，最后会有90M，171个jar包

图片25

---

• 右键工程最后一个选项，进入Properties for hdfs -> java build path

图24

---

• Libraries->Add Extemal Jar，找到/usr/local/hadoop/share/hadoop/alllib全选jar包，ok
• 会有一个增加for的错误，直接鼠标点修改jre1.5以上，自动改，这个跟idea一样的

图片26

---

• 全部正确，准备运行

图片27

测试Hadoop API

public static void main(String[] args) throws Exception {
    //上传文件到hadoop
   uploadFile();
   createFile();
   createDir();
   fileRename();
   deleteFile();
   readFile();
   isFileExists();
    //判断文件的最后修改的信息
   fileLastModify();
    //文件的位置信息
   fileLocation();
    //节点的信息
   nodeList();
}

使用HDFS文件系统并提供服务器路径

static FileSystem getFileSystem() throws Exception {
   URI uri = new URI("hdfs://hadoop0:9000/");
   // 使用HDFS文件系统并提供服务器路径，端口号在core-site.xml中配置
   FileSystem fileSystem = FileSystem.get(uri, new Configuration());
   return fileSystem;
}

• 测试getFileSystem

图片28

run as->java appliction->ok

图片29

其他api不一一实现了，很简单的，就是JDBC最简单的那种

Code

package zuosheng.Binarytree;


import java.net.URI;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.io.IOUtils;

public class Hdfs {

   /**
    * @param args
    * @throws Exception
    */
   public static void main(String[] args) throws Exception {
      uploadFile();
      createFile();
      createDir();
      fileRename();
      deleteFile();
      readFile();
      isFileExists();
      fileLastModify();
      fileLocation();
      nodeList();
   }

   static FileSystem getFileSystem() throws Exception {
      URI uri = new URI("hdfs://hadoop0:9000/");
      // 使用HDFS文件系统并提供服务器路径，端口号在core-site.xml中配置
      FileSystem fileSystem = FileSystem.get(uri, new Configuration());
      return fileSystem;
   }

   public static void uploadFile() throws Exception {

      FileSystem hdfs = getFileSystem();
      Path src = new Path("/root/original-ks.cfg");
      Path dst = new Path("/");
      FileStatus files[] = hdfs.listStatus(dst);
      for (FileStatus file : files) {
         System.out.println(file.getPath());
      }
      System.out.println("------------after upload--------------------");
      hdfs.copyFromLocalFile(src, dst);
      files = hdfs.listStatus(dst);
      for (FileStatus file : files) {
         System.out.println(file.getPath());
      }
   }

   public static void createFile() throws Exception {

      byte[] buff = "Hello Hadoop 888@Chinasofti\n".getBytes();
      FileSystem hdfs = getFileSystem();
      Path dfs = new Path("/testcreate");
      FSDataOutputStream outputStream = hdfs.create(dfs);
      outputStream.write(buff, 0, buff.length);
      outputStream.close();

   }

   public static void createDir() throws Exception {
      FileSystem hdfs = getFileSystem();
      Path dfs = new Path("/TestDir");
      hdfs.mkdirs(dfs);
   }

   public static void fileRename() throws Exception {

      FileSystem hdfs = getFileSystem();
      Path frpaht = new Path("/original-ks.cfg");
      Path topath = new Path("/original-ks2.cfg");
      boolean isRename = hdfs.rename(frpaht, topath);
      String result = isRename ? "成功" : "失败";
      System.out.println("文件重命名结果为：" + result);
   }

   public static void deleteFile() throws Exception {
      FileSystem hdfs = getFileSystem();
      Path delef = new Path("/TestDir");
      boolean isDeleted = hdfs.delete(delef, false);
      // 递归删除
      // boolean isDeleted=hdfs.delete(delef,true);
      System.out.println("Delete?" + isDeleted);
   }

   public static void readFile() throws Exception {
      FileSystem fileSystem = getFileSystem();
      FSDataInputStream openStream = fileSystem.open(new Path("/testcreate"));
      IOUtils.copyBytes(openStream, System.out, 1024, false);
      IOUtils.closeStream(openStream);

   }

   public static void isFileExists() throws Exception {
      FileSystem hdfs = getFileSystem();
      Path findf = new Path("/test1");
      boolean isExists = hdfs.exists(findf);
      System.out.println("Exist?" + isExists);
   }

   public static void fileLastModify() throws Exception {
      FileSystem hdfs = getFileSystem();
      Path fpath = new Path("/testcreate");
      FileStatus fileStatus = hdfs.getFileStatus(fpath);
      long modiTime = fileStatus.getModificationTime();
      System.out.println("testcreate的修改时间是" + modiTime);
   }

   public static void fileLocation() throws Exception {
      FileSystem hdfs = getFileSystem();
      Path fpath = new Path("/testcreate");
      FileStatus filestatus = hdfs.getFileStatus(fpath);
      BlockLocation[] blkLocations = hdfs.getFileBlockLocations(filestatus,
            0, filestatus.getLen());
      int blockLen = blkLocations.length;
      for (int i = 0; i < blockLen; i++) {
         String[] hosts = blkLocations[i].getHosts();
         System.out.println("block_" + i + "_location:" + hosts[0]);
      }

   }

   public static void nodeList() throws Exception {
      FileSystem fs = getFileSystem();
      DistributedFileSystem hdfs = (DistributedFileSystem) fs;

      DatanodeInfo[] dataNodeStats = hdfs.getDataNodeStats();
      for (int i = 0; i < dataNodeStats.length; i++) {
         System.out.println("DataNode_" + i + "_Name:"
               + dataNodeStats[i].getHostName());

      }

   }

}