maven+hadoop2.2 项目HDFS文件读取程序

前提已经安装配置好hadoop2.2，安装请参考我另一篇文章

排版太辛苦，先弄个简易版出来,只要看显示的那两个类就够了，其他部分是我项目无关HDFS文件读写（请见谅）

pom.xml配置如下

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

<modelVersion>4.0.0</modelVersion>

<parent>

<artifactId>storage</artifactId>

<groupId>platform.edu.storage</groupId>

<version>0.0.1-SNAPSHOT</version>

</parent>

<artifactId>explorer-impl</artifactId>

<name>${project.artifactId}</name>

<packaging>jar</packaging>

<description>Hadoop HDFS explorer实现</description>

<dependencies>

<!-- hadoop base before -->

<dependency>

<groupId>org.apache.hadoop</groupId>

<artifactId>hadoop-client</artifactId>

<version>2.2.0</version>

</dependency>

<!-- hadood base end -->

</dependencies>

HDFSServer.java代码如下

-----------------------------------------------------------

package platform.edu.explorer.hdfs.server;

 

import java.io.IOException;

import java.net.URI;

 

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.mapred.JobConf;

 

/**

 * HDFS服务器

 * 

 * @author hjn

 * @version 1.0 2013-11-20

 */

public class HDFSServer {

private static Configuration configuration;

private static FileSystem fileSystem;

private static final String HDFS_URL = "hdfs://192.168.1.210:8020";

 

/**

 * HDFS服务器读取初始化

 */

private static void init() {

try {

configuration= new JobConf(HDFSServer.class);

fileSystem = FileSystem.get(URI.create(HDFS_URL), configuration);

} catch (IOException e) {

System.out.println("读取服务器失败");

e.printStackTrace();

}

}

public static FileSystem getFileSystem(){

if(fileSystem==null){

init();

}

return fileSystem;

}

 

}

 

--------------------------------------------------------------------------------------------------------------------------------------------------
HDFS.java代码如下

--------------------------

package platform.edu.explorer.hdfs;

 

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;

import java.util.ArrayList;

import java.util.List;

 

import org.apache.commons.compress.utils.IOUtils;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

 

import platform.edu.explorer.HadoopFileSystem;

import platform.edu.explorer.hdfs.server.HDFSServer;

 

/**

 * HDFS(分布式文件存储系统操作类).

 * 

 * @author hjn 

 * @version 1.0 2013-11-22

 */

 

public class HDFS implements HadoopFileSystem {

/**

 * 文件系统类

 */

private FileSystem fileSystem;

 

/**

 * 无参数构造方法

 */

public HDFS() {

   init();

}

 

/**

 * 初始化

 */

private void init() {

   fileSystem = HDFSServer.getFileSystem();

}

 

/**

 * 获取HDFS指定目录下文件状态列表

 * 

 * @param dirPath指定目录路径

 * @return fileStatusList

 * @throws FileNotFoundException

 * @throws IOException

 */

public FileStatus[] getFileStatus(Path path) throws FileNotFoundException,

IOException {

   FileStatus[] fileStatusList = fileSystem.listStatus(path);

   return fileStatusList;

}

 

/**

 * 获取指定目录列表路径

 * 

 * @param dirPath

 */

public List<String> dir(String dirPath) throws IOException {

  List<String> fileList = null;

  Path path = new Path(dirPath);

  if (fileSystem.exists(path)) {

    fileList = new ArrayList<String>();

    FileStatus[] list = this.getFileStatus(path);

  for (FileStatus fileStatus : list) {

    fileList.add(fileStatus.getPath().toString());

  }

  } else {

     System.out.println("目录不存在");

  }

  return fileList;

}

 

/**

 * 获取文件

 * 

 * @param filePath

 * @return

 * @throws IOException

 */

public InputStream getFile(String filePath) throws IOException {

Path path = new Path(filePath);

return fileSystem.open(path);

}

 

/**

 * 更改HDSF文件名称

 * 

 * @param fileOldName

 * @param fileNewName

 * @return boolean:是否更名字成功

 * @throws IOException

 */

public boolean rename(String src, String dst) throws IOException {

   Path srcPath = new Path(src);

   if (fileSystem.exists(srcPath)) {

     Path dstPath = new Path(dst);

     return fileSystem.rename(srcPath, dstPath);

   }

   System.out.println("原文件不存在");

   return false;

}

 

/**

 * 创建HDFS目录

 * 

 * @param dir

 */

public boolean createDir(String dir) throws IOException {

   Path path = new Path(dir);

   if (fileSystem.exists(path)) {

     System.out.println("此目录已经存在不需要再创建");

   return true;

   }

   return fileSystem.mkdirs(path);

}

 

/**

 * 上传本地文件到HDFS（注意是服务器本地硬盘，非客户端硬盘)）

 * 

 * @return

 * 

 * @throws IOException

 */

@Override

public void uploadLocalFile(String localFileSrc, String HDFSFileDst)

           throws IOException {

  Path src = new Path(localFileSrc);

  Path dst = new Path(HDFSFileDst);

  fileSystem.copyFromLocalFile(src, dst);

}

/**

 * 批量上传本地文件到HDFS

 * @param localFileSrcs本地文件列表

 * @param HDFSFileDst

 * @throws IOException

 */

    public void uploadLocalFile(String[] localFileSrcs,String HDFSFileDst) throws IOException{

    Path dstPath=new Path(HDFSFileDst);

    Path[] paths=new Path[localFileSrcs.length];

    for (int i=0;i<localFileSrcs.length;i++) {

paths[i]=new Path(localFileSrcs[i]);

}

    fileSystem.copyFromLocalFile(false, false, paths,dstPath);

    }

/**

 * 从HDFS下载文件到本地(注意是服务器本地硬盘，非浏览器客户端硬盘)

 * 

 * @param HDFSFilePath

 * @param localFilePath

 * @throws IOException

 */

public void downFileToLocal(String HDFSFilePath, String localFilePath)

throws IOException {

  Path dstPath = new Path(HDFSFilePath);

  FSDataInputStream in = fileSystem.open(dstPath);

  OutputStream out = new FileOutputStream(new File(localFilePath));

  IOUtils.copy(in, out);

}

 

/**

 * HDFS文件是否存在

 * 

 * @param filePath

 * @return boolean:是否存在

 * @throws IOException

 */

public boolean exists(String filePath) throws IOException {

  Path path = new Path(filePath);

  return fileSystem.exists(path);

}

 

/**

 * 根据路径删除文件或文件夹

 * 

 * @param filePath

 * @return

 * @throws IOException

 */

@SuppressWarnings("deprecation")

public boolean deleteFile(String filePath) throws IOException {

  if (this.exists(filePath)) {

    Path path = new Path(filePath);

    fileSystem.delete(path);

    return true;

  }

  System.out.println("文件不存在");

  return false;

}

 

/**

 * 剪切本地文件到HDFS(注意为服务器本地文件);

 * 

 * @param src本地路径

 * @param dst分布式存储路径

 * @throws IOException

 */

public void moveFromLocalFile(String localSrc, String HDFSDst) throws IOException {

   Path srcPath = new Path(localSrc);

   Path dstPath = new Path(HDFSDst);

   fileSystem.moveFromLocalFile(srcPath, dstPath);

}

/**

 * HDFS文件之间的复制

 * @param src源文件路径

 * @param dst要复制后复制文件的路径

 * @throws IOException

 */

public void copyHDFSFile(String src,String dst) throws IOException{

   Path srcPath=new Path(src);

   Path dstPath=new Path(dst);

   InputStream in=fileSystem.open(srcPath);

   OutputStream out=fileSystem.create(dstPath);

   IOUtils.copy(in, out);

}

/**

 * HDFS中移动文件

 * @param src源文件路径

 * @param dst要移动后的路径

 * @throws IOException

 */

@SuppressWarnings("deprecation")

public void moveHDFSFile(String src,String dst) throws IOException{

   Path srcPath=new Path(src);

   Path dstPath=new Path(dst);

   InputStream in=fileSystem.open(srcPath);

   OutputStream out=fileSystem.create(dstPath);

   IOUtils.copy(in, out);

   fileSystem.delete(srcPath);

}

/**

 * 剪切HDFS文件到本地

 * @param HDFSSrc

 * @param localDst

 * @throws IOException

 */

    public void moveToLocalFile(String HDFSSrc,String localDst) throws IOException{

    Path srcPath = new Path(HDFSSrc);

     Path dstPath = new Path(localDst);

    fileSystem.moveToLocalFile(srcPath, dstPath);

    }

/**

 * HDFS创建文件

 * 

 * @param in输入流

 * @param dst分布式存储路径

 * @throws IOException

 */

public void create(InputStream in, String dst) throws IOException {

   Path dstPath = new Path(dst);

   FSDataOutputStream out = fileSystem.create(dstPath);

   IOUtils.copy(in, out);

}

 

/**

 * 在HDFS创建文件

 * 

 * @param file

 * @param dst分布式存储路径

 * @throws IOException

 */

public void create(File file, String dst) throws IOException {

   InputStream in = new FileInputStream(file);

   this.create(in, dst);

}

 

/**

 * 在HDFS创建文件

 * 

 * @param src本地文件路径

 * @param dst分布式存储路径

 * @throws IOException

 */

public void create(String src, String dst) throws IOException {

   File file = new File(src);

   this.create(file, dst);

}

 

/**

 * 获取FileSystem对象

 * 

 * @return

 */

public FileSystem getFileSystem() {

   return fileSystem;

}

 

/**

 * 关闭HDFS

 * 

 * @throws IOException

 */

public void close() throws IOException {

   fileSystem.close();

}

 

public boolean isDir(String src) throws IOException {

   Path path = new Path(src);

   return fileSystem.isDirectory(path);

}

 

public static void main(String[] args) throws IOException {

      HDFS hdfs = 
    new HDFS();

    // hdfs.uploadLocalFile("D:/picture", "/test"); //

      System.
    out.println(hdfs.dir("/"));

    // hdfs.create("D:/picture/mypicture/POP海报2590.jpg","/test/picture/mypicture/POP海报2590.jpg");

      System.
    out.println(hdfs.dir("/test/picture/mypicture"));

    //hdfs.uploadLocalFile(new String[]{"E:/input","E:/output"}, "/");

    //hdfs.rename("/input", "/debug_in");

    System.
    out.println(hdfs.dir("/debug_out"));

    //hdfs.deleteFile("/output");

    // hdfs.moveFromLocalFile("E:/test.jpg", "/test/picture/POP海报2590.jpg");

    /*

     * System.out.println(hdfs.dir("/test"));

     * System.out.println(hdfs.exists("/test/picture/mypicture")); //

     * hdfs.delete("/test/picture/mypicture");

     * System.out.println(hdfs.dir("/test/picture"));

     */

    hdfs.close();

}

}