java 远程 hdfs_java远程操作HDFS

java远程操作hadoop的分布式文件系统hdfs

需要导入的hadoop的jar包

hadoop解压包中有所需的jar包

hadoop-2.5.2/share/hadoop/common/hadoop-common-2.5.2.jar

hadoop-2.5.2/share/hadoop/common/lib/*.jar(lib下的全部jar包)

hadoop-2.5.2/share/hadoop/hdfs/hadoop-hdfs-2.5.2.jar

如果是集群内部用eclipse中的java程序访问hdfs的话，可以直接写代码操作，如果是远程操作的话需要配置hdfs-site.xml文件，关闭权限检查，不配置的话会报错Permission denied

hdfs-site.xml

dfs.permissions.enabled

false

import java.io.FileInputStream;

import java.io.IOException;

import java.io.InputStream;

import java.net.MalformedURLException;

import java.net.URL;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IOUtils;

/**

* java远程或本地操作hdfs文件系统(eclipse)

* @author root

*

*/

public class HDFSFile {

public static void main(String [] args) throws IOException {

/**

* java读取hdfs文件信息

*/

/* 方法一 */

URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());

URL url = new URL("hdfs://master:9000/hello.txt");

InputStream in = url.openStream();

IOUtils.copyBytes(in,System.out,4096,true);//in是输入流，System.out是标准输出，4096是指定缓存区大小，true是读完后把输入流自动关闭

/* 方法二 */

Configuration conf = new Configuration();

conf.set("fs.defaultFS","hdfs://master:9000");

FileSystem filesystem = FileSystem.get(conf);

boolean success = filesystem.mkdirs(new Path("/test"));

System.out.println(success);

// 判断文件夹是否存在

success = filesystem.exists(new Path("/hello.txt"));

System.out.println(success);

// 删除文件夹或文件

success = filesystem.delete(new Path("/test"),true);

System.out.println(success);

// true --> now false --> delete home

success = filesystem.exists(new Path("/test"));

System.out.println(success);

// 向hdfs文件中写入数据

FSDataOutputStream out = filesystem.create(new Path("/test.data"),true);

FileInputStream in = new FileInputStream("/root/a.txt");

byte [] buf = new byte[4096];

int len = in.read(buf);

while (len != -1) {

out.write(buf,0,len);

len = in.read(buf);

}

in.close();

out.close();

// 查看文件hdfs文件目录

FileStatus[] fs = filesystem.listStatus(new Path("/"));

for(FileStatus status :fs) {

System.out.println(status.getPath());

System.out.println(status.getPermission());

System.out.println(status.getReplication());//copy how much

}

}

}