前提已经安装配置好hadoop2.2,安装请参考我另一篇文章
排版太辛苦,先弄个简易版出来,只要看显示的那两个类就够了,其他部分是我项目无关HDFS文件读写(请见谅)
pom.xml配置如下
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>storage</artifactId>
<groupId>platform.edu.storage</groupId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<artifactId>explorer-impl</artifactId>
<name>${project.artifactId}</name>
<packaging>jar</packaging>
<description>Hadoop HDFS explorer实现</description>
<dependencies>
<!-- hadoop base before -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.2.0</version>
</dependency>
<!-- hadood base end -->
</dependencies>
HDFSServer.java代码如下
-----------------------------------------------------------
package platform.edu.explorer.hdfs.server;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf;
/**
* HDFS服务器
*
* @author hjn
* @version 1.0 2013-11-20
*/
public class HDFSServer {
private static Configuration configuration;
private static FileSystem fileSystem;
private static final String HDFS_URL = "hdfs://192.168.1.210:8020";
/**
* HDFS服务器读取初始化
*/
private static void init() {
try {
configuration= new JobConf(HDFSServer.class);
fileSystem = FileSystem.get(URI.create(HDFS_URL), configuration);
} catch (IOException e) {
System.out.println("读取服务器失败");
e.printStackTrace();
}
}
public static FileSystem getFileSystem(){
if(fileSystem==null){
init();
}
return fileSystem;
}
}
--------------------------------------------------------------------------------------------------------------------------------------------------
HDFS.java代码如下
--------------------------
package platform.edu.explorer.hdfs;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import platform.edu.explorer.HadoopFileSystem;
import platform.edu.explorer.hdfs.server.HDFSServer;
/**
* HDFS(分布式文件存储系统操作类).
*
* @author hjn
* @version 1.0 2013-11-22
*/
public class HDFS implements HadoopFileSystem {
/**
* 文件系统类
*/
private FileSystem fileSystem;
/**
* 无参数构造方法
*/
public HDFS() {
init();
}
/**
* 初始化
*/
private void init() {
fileSystem = HDFSServer.getFileSystem();
}
/**
* 获取HDFS指定目录下文件状态列表
*
* @param dirPath指定目录路径
* @return fileStatusList
* @throws FileNotFoundException
* @throws IOException
*/
public FileStatus[] getFileStatus(Path path) throws FileNotFoundException,
IOException {
FileStatus[] fileStatusList = fileSystem.listStatus(path);
return fileStatusList;
}
/**
* 获取指定目录列表路径
*
* @param dirPath
*/
public List<String> dir(String dirPath) throws IOException {
List<String> fileList = null;
Path path = new Path(dirPath);
if (fileSystem.exists(path)) {
fileList = new ArrayList<String>();
FileStatus[] list = this.getFileStatus(path);
for (FileStatus fileStatus : list) {
fileList.add(fileStatus.getPath().toString());
}
} else {
System.out.println("目录不存在");
}
return fileList;
}
/**
* 获取文件
*
* @param filePath
* @return
* @throws IOException
*/
public InputStream getFile(String filePath) throws IOException {
Path path = new Path(filePath);
return fileSystem.open(path);
}
/**
* 更改HDSF文件名称
*
* @param fileOldName
* @param fileNewName
* @return boolean:是否更名字成功
* @throws IOException
*/
public boolean rename(String src, String dst) throws IOException {
Path srcPath = new Path(src);
if (fileSystem.exists(srcPath)) {
Path dstPath = new Path(dst);
return fileSystem.rename(srcPath, dstPath);
}
System.out.println("原文件不存在");
return false;
}
/**
* 创建HDFS目录
*
* @param dir
*/
public boolean createDir(String dir) throws IOException {
Path path = new Path(dir);
if (fileSystem.exists(path)) {
System.out.println("此目录已经存在不需要再创建");
return true;
}
return fileSystem.mkdirs(path);
}
/**
* 上传本地文件到HDFS(注意是服务器本地硬盘,非客户端硬盘))
*
* @return
*
* @throws IOException
*/
@Override
public void uploadLocalFile(String localFileSrc, String HDFSFileDst)
throws IOException {
Path src = new Path(localFileSrc);
Path dst = new Path(HDFSFileDst);
fileSystem.copyFromLocalFile(src, dst);
}
/**
* 批量上传本地文件到HDFS
* @param localFileSrcs本地文件列表
* @param HDFSFileDst
* @throws IOException
*/
public void uploadLocalFile(String[] localFileSrcs,String HDFSFileDst) throws IOException{
Path dstPath=new Path(HDFSFileDst);
Path[] paths=new Path[localFileSrcs.length];
for (int i=0;i<localFileSrcs.length;i++) {
paths[i]=new Path(localFileSrcs[i]);
}
fileSystem.copyFromLocalFile(false, false, paths,dstPath);
}
/**
* 从HDFS下载文件到本地(注意是服务器本地硬盘,非浏览器客户端硬盘)
*
* @param HDFSFilePath
* @param localFilePath
* @throws IOException
*/
public void downFileToLocal(String HDFSFilePath, String localFilePath)
throws IOException {
Path dstPath = new Path(HDFSFilePath);
FSDataInputStream in = fileSystem.open(dstPath);
OutputStream out = new FileOutputStream(new File(localFilePath));
IOUtils.copy(in, out);
}
/**
* HDFS文件是否存在
*
* @param filePath
* @return boolean:是否存在
* @throws IOException
*/
public boolean exists(String filePath) throws IOException {
Path path = new Path(filePath);
return fileSystem.exists(path);
}
/**
* 根据路径删除文件或文件夹
*
* @param filePath
* @return
* @throws IOException
*/
@SuppressWarnings("deprecation")
public boolean deleteFile(String filePath) throws IOException {
if (this.exists(filePath)) {
Path path = new Path(filePath);
fileSystem.delete(path);
return true;
}
System.out.println("文件不存在");
return false;
}
/**
* 剪切本地文件到HDFS(注意为服务器本地文件);
*
* @param src本地路径
* @param dst分布式存储路径
* @throws IOException
*/
public void moveFromLocalFile(String localSrc, String HDFSDst) throws IOException {
Path srcPath = new Path(localSrc);
Path dstPath = new Path(HDFSDst);
fileSystem.moveFromLocalFile(srcPath, dstPath);
}
/**
* HDFS文件之间的复制
* @param src源文件路径
* @param dst要复制后复制文件的路径
* @throws IOException
*/
public void copyHDFSFile(String src,String dst) throws IOException{
Path srcPath=new Path(src);
Path dstPath=new Path(dst);
InputStream in=fileSystem.open(srcPath);
OutputStream out=fileSystem.create(dstPath);
IOUtils.copy(in, out);
}
/**
* HDFS中移动文件
* @param src源文件路径
* @param dst要移动后的路径
* @throws IOException
*/
@SuppressWarnings("deprecation")
public void moveHDFSFile(String src,String dst) throws IOException{
Path srcPath=new Path(src);
Path dstPath=new Path(dst);
InputStream in=fileSystem.open(srcPath);
OutputStream out=fileSystem.create(dstPath);
IOUtils.copy(in, out);
fileSystem.delete(srcPath);
}
/**
* 剪切HDFS文件到本地
* @param HDFSSrc
* @param localDst
* @throws IOException
*/
public void moveToLocalFile(String HDFSSrc,String localDst) throws IOException{
Path srcPath = new Path(HDFSSrc);
Path dstPath = new Path(localDst);
fileSystem.moveToLocalFile(srcPath, dstPath);
}
/**
* HDFS创建文件
*
* @param in输入流
* @param dst分布式存储路径
* @throws IOException
*/
public void create(InputStream in, String dst) throws IOException {
Path dstPath = new Path(dst);
FSDataOutputStream out = fileSystem.create(dstPath);
IOUtils.copy(in, out);
}
/**
* 在HDFS创建文件
*
* @param file
* @param dst分布式存储路径
* @throws IOException
*/
public void create(File file, String dst) throws IOException {
InputStream in = new FileInputStream(file);
this.create(in, dst);
}
/**
* 在HDFS创建文件
*
* @param src本地文件路径
* @param dst分布式存储路径
* @throws IOException
*/
public void create(String src, String dst) throws IOException {
File file = new File(src);
this.create(file, dst);
}
/**
* 获取FileSystem对象
*
* @return
*/
public FileSystem getFileSystem() {
return fileSystem;
}
/**
* 关闭HDFS
*
* @throws IOException
*/
public void close() throws IOException {
fileSystem.close();
}
public boolean isDir(String src) throws IOException {
Path path = new Path(src);
return fileSystem.isDirectory(path);
}
public static void main(String[] args) throws IOException {
HDFS hdfs =
new HDFS();
// hdfs.uploadLocalFile("D:/picture", "/test"); //
System.
out.println(hdfs.dir("/"));
// hdfs.create("D:/picture/mypicture/POP海报2590.jpg","/test/picture/mypicture/POP海报2590.jpg");
System.
out.println(hdfs.dir("/test/picture/mypicture"));
//hdfs.uploadLocalFile(new String[]{"E:/input","E:/output"}, "/");
//hdfs.rename("/input", "/debug_in");
System.
out.println(hdfs.dir("/debug_out"));
//hdfs.deleteFile("/output");
// hdfs.moveFromLocalFile("E:/test.jpg", "/test/picture/POP海报2590.jpg");
/*
* System.out.println(hdfs.dir("/test"));
* System.out.println(hdfs.exists("/test/picture/mypicture")); //
* hdfs.delete("/test/picture/mypicture");
* System.out.println(hdfs.dir("/test/picture"));
*/
hdfs.close();
}
}