Java开发~Eclipse访问集群hdfs的配置和参考

·1、准备工作

将hadoop-eclipse-plugin-2.6.0.jar放于eclipse的plugin目录下

(hadoop-eclipse-kepler-plugin-2.2.0.jar、hadoop-eclipse-kepler-plugin-2.4.1.jar也建议放进去)

2、配置

解压hadoop-2.6.0-cdh5.4.7.tar到目录下，配置环境变量如下：

3、相关依赖

将以下几个文件放到%HADOOP_HOME%\bin目录下：

4、配置文件

将集群中的配置文件放到%HADOOP_HOME%\conf目录下，主要文件如下：

5、创建工程

启动eclipse，建立MapReduce Project，程序demo代码如下：

public class accessHDFS {
	private static final Log log = LogFactory.getLog(accessHDFS.class);

	public static void main(String[] args) throws Exception {

		Configuration conf = new Configuration();
		conf.set("hadoop.security.authentication", "Kerberos");
		System.setProperty("java.security.krb5.realm", "BIGDATAS.COM");
		System.setProperty("java.security.krb5.kdc", "secnamenode");
		UserGroupInformation.setConfiguration(conf);
        //keytab文件及用户名根据分配权限修改
		UserGroupInformation.loginUserFromKeytab("jia@BIGDATAS.COM",
				"F:\\hadoop\\hadoop-2.6.0\\conf\\jia.keytab");

		conf.set("fs.default.name", "hdfs://namenode:8020");
		FileSystem fs = FileSystem.get(conf);
		Path path = new Path("/user/jia/write111.txt");
		//文件列表
		listDir(conf,"/user");
		//读取文件内容
		/*FileSystem hdfs = FileSystem.get(conf);
		  InputStream in = null;
		  try{
		  in = hdfs.open(path);
		  IOUtils.copyBytes(in,System.out,4096,true);
		  }finally{
		  IOUtils.closeStream(in);
		  }*/
		//写文件
		/*
		 * FSDataOutputStream out = fs.create(path); out.writeBytes(
		 * "da jia hao,cai shi zhen de hao!\nda jia hao,cai shi zhen de hao");
		 * fs.close();
		 */
        //读文件
		/*if (fs.exists(path)) {
			FSDataInputStream is = fs.open(path);
			FileStatus status = fs.getFileStatus(path);
			byte[] buffer = new byte[Integer.parseInt(String.valueOf(status.getLen()))];
			is.readFully(0, buffer);
			System.out.println(String.valueOf(status.getLen()));
			is.close();
			fs.close();
			System.out.println(buffer.toString());
		}*/
	}

	public static void listDir(Configuration conf, String dir) throws IOException {
		FileSystem fs = FileSystem.get(conf);
		FileStatus files[] = fs.listStatus(new Path(dir));
		for (FileStatus file : files) {
			System.out.println(file.getPath());
		}
	}

}

6、执行

在eclipse中添加环境变量如下，value即conf所在目录：

欢迎关注交流：