Java操作HDFS(常用API)

先前准备

• 确保HDFS集群已经搭建 如有不会请查看分布式HDFS集群搭建
• 在本机配好hadoop环境变量，必要时在hosts文件下加入node和ip
• 在eclipse中配置Hadoop插件
  • 在eclipse安装目录下打开dropins文件夹新建plugins放入hadoop-eclipse-plugin-2.6.0.jar
    
• 导入依赖的jar包(在hadoop安装包下)

API操作

• 首先先创建配置对象

// 操作HDFS之前得先创建配置对象
Configuration conf = new Configuration(true);
// 创建操作HDFS的对象
FileSystem fs=fileSystem.get(conf);

查看文件

// 调用listStatus(Path path)方法
private static List listFileSystem(FileSystem fs, String path) throws FileNotFoundException, IOException {
		Path ppath = new Path(path);// 获取路径
		FileStatus[] listStatus = fs.listStatus(ppath);
		
		for (FileStatus fileStatus : listStatus) {
			System.out.println(fileStatus.getPath());
		}
		return null;
	}

创建新文件夹

private static void createDir(FileSystem fs, String string) 				throws IllegalArgumentException, IOException {
	Path path = new Path(string);
	if(fs.exists(path)){// 判断文件是否存在
		fs.delete(path, true);
	}	
	fs.mkdirs(path);
}

上传文件

// copyFromLocalFile()
private static void uploadFileToHDFS(FileSystem fs, String src, String dest) throws IOException {
	Path srcPath = new Path(src);
	Path destPath = new Path(dest);
	//copyFromLocal 复制
	fs.copyFromLocalFile(srcPath, destPath);
	//moveFromLocal 剪切
	fs.copyFromLocalFile(true,srcPath, destPath);
}

下载文件

private static void downLoadFileFromHDFS(FileSystem fs, String src, String dest) throws IOException {
	Path srcPath = new Path(src);
	Path destPath = new Path(dest);
	//copyToLocal 复制到本地
//	fs.copyToLocalFile(srcPath, destPath);
	//moveToLocal 剪切到本地
	fs.copyToLocalFile(true,srcPath, destPath);
}

删除文件

fs.delete(path,true);

内部复制和内部移动(剪切)

private static void innerCopyAndMoveFile(FileSystem fs, Configuration conf,String src, String dest) throws IOException {
	Path srcPath = new Path(src);
	Path destPath = new Path(dest);	
	//内部复制
	FileUtil.copy(srcPath.getFileSystem(conf), srcPath, destPath.getFileSystem(conf), destPath,false, conf);
	//内部移动
	FileUtil.copy(srcPath.getFileSystem(conf), srcPath, destPath.getFileSystem(conf), destPath,true, conf);
}

重命名

private static void renameFile(FileSystem fs, String src, String dest) throws IOException {
	Path srcPath = new Path(src);
	Path destPath = new Path(dest);	
	fs.rename(srcPath, destPath);
	}

创建新的文件

private static void createNewFile(FileSystem fs, String string) throws IllegalArgumentException, IOException {
	fs.createNewFile(new Path(string));
}

写文件

private static void writeToHDFSFile(FileSystem fs, String filePath, String content) throws IllegalArgumentException, IOException {
	FSDataOutputStream outputStream = fs.create(new Path(filePath));
	outputStream.write(content.getBytes("UTF-8"));
	outputStream.flush();
	outputStream.close();
}

读文件

private static void readFromHDFSFile(FileSystem fs, String string) throws IllegalArgumentException, IOException {
	FSDataInputStream inputStream = fs.open(new Path(string));	
	FileStatus fileStatus = fs.getFileStatus(new Path(string));
	long len = fileStatus.getLen();	
		byte[] b = new byte[(int)len];
		int read = inputStream.read(b);
		while(read != -1){
			System.out.println(new String(b));
			read = inputStream.read(b);
		}
}

追加写

private static void appendToHDFSFile(FileSystem fs, String filePath, String content) throws IllegalArgumentException, IOException {
	FSDataOutputStream append = fs.append(new Path(filePath));
	append.write(content.getBytes("UTF-8"));
	append.flush();
	append.close();
}

获取数据位置

private static void getFileLocation(FileSystem fs, String string) throws IOException {
	FileStatus fileStatus = fs.getFileStatus(new Path(string));
	long len = fileStatus.getLen();
	BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, len);
	String[] hosts = fileBlockLocations[0].getHosts();
	for (String string2 : hosts) {
		System.out.println(string2);
	}	
	HdfsBlockLocation blockLocation = (HdfsBlockLocation)fileBlockLocations[0];
	long blockId = blockLocation.getLocatedBlock().getBlock().getBlockId();
	System.out.println(blockId);
}