hdfs java api读写

如果想在win下下载hdfs的文件，需要安装hadoop的win版，因为写入磁盘的程序对linux和win编译结果不同
下载地址是：
https://github.com/cdarlint/winutils

常用方法	描述
copyFromLocalFile	从本地上传文件
copyToLocalFile	从hdfs读取文件,在win系统中要用hadoop中的相应文件
rename	移动和重命名
mkdirs	创建文件夹
delete
listFiles
listStatus
open	读取文件
create	写入文件

package cn.edu360.hdfs.demo;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

public class HdfsClientDemo {

    public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
        //new Configura()会自动加载core-default.xml,hdfs-default.xml,yarn-default.xml等文件
        //1.首先加载jar包里的xml默认参数
        //2.再加载用户在src创建的同名xml参数
        //3.最后加载conf.set()参数
        Configuration conf = new Configuration();//配置文件
        conf.set("dfs.replication","2");//覆盖默认xml文件配置
        conf.set("dfs.blocksize","64m");
        //uri是指namenode的端口,conf是配置,user是linux的用户,默认是客户端的用户
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.56.1:9000/"), conf, "root");
        fs.copyFromLocalFile(new Path("D:\\download\\PDFXVwer.zip"),new Path("/aaa/PDFXVwer.zip"));
        fs.close();
    }

	FileSystem fs = null;

	@Before
	public void init() throws Exception {
		Configuration conf = new Configuration();
		conf.set("dfs.replication", "2");
		conf.set("dfs.blocksize", "64m");

		fs = FileSystem.get(new URI("hdfs://master:9000/"), conf, "root");

	}

	/**
	 * 从HDFS中下载文件到客户端本地磁盘
	 * 要下载一个hadoopwin下的写文件的程序
	 * @throws IOException
	 * @throws IllegalArgumentException
	 */
	@Test
	public void testGet() throws IllegalArgumentException, IOException {

		fs.copyToLocalFile(new Path("/xxx.txt"), new Path("f:/"));
		fs.close();

	}

	/**
	 * 在hdfs内部移动文件\修改名称
	 */
	@Test
	public void testRename() throws Exception {

		fs.rename(new Path("/install.log"), new Path("/aaa/in.log"));

		fs.close();

	}

	/**
	 * 在hdfs中创建文件夹
	 */
	@Test
	public void testMkdir() throws Exception {

		fs.mkdirs(new Path("/xx/yy/zz"));

		fs.close();
	}

	/**
	 * 在hdfs中删除文件或文件夹
	 */
	@Test
	public void testRm() throws Exception {

		fs.delete(new Path("/aaa"), true);

		fs.close();
	}

	/**
	 * 查询hdfs指定目录下的文件信息
	 */
	@Test
	public void testLs() throws Exception {
		// 只查询文件的信息,不返回文件夹的信息
		RemoteIterator<LocatedFileStatus> iter = fs.listFiles(new Path("/"), true);

		while (iter.hasNext()) {
			LocatedFileStatus status = iter.next();
			System.out.println("文件全路径：" + status.getPath());
			System.out.println("块大小：" + status.getBlockSize());
			System.out.println("文件长度：" + status.getLen());
			System.out.println("副本数量：" + status.getReplication());
			System.out.println("块信息：" + Arrays.toString(status.getBlockLocations()));

			System.out.println("--------------------------------");
		}
		fs.close();
	}

	/**
	 * 查询hdfs指定目录下的文件和文件夹信息
	 */
	@Test
	public void testLs2() throws Exception {
		FileStatus[] listStatus = fs.listStatus(new Path("/"));

		for (FileStatus status : listStatus) {
			System.out.println("文件全路径：" + status.getPath());
			System.out.println(status.isDirectory() ? "这是文件夹" : "这是文件");
			System.out.println("块大小：" + status.getBlockSize());
			System.out.println("文件长度：" + status.getLen());
			System.out.println("副本数量：" + status.getReplication());

			System.out.println("--------------------------------");
		}
		fs.close();
	}

	/**
	 * 读取hdfs中的文件的内容
	 * 
	 * @throws IOException
	 * @throws IllegalArgumentException
	 */
	@Test
	public void testReadData() throws IllegalArgumentException, IOException {

		FSDataInputStream in = fs.open(new Path("/test.txt"));

		BufferedReader br = new BufferedReader(new InputStreamReader(in, "utf-8"));

		String line = null;
		while ((line = br.readLine()) != null) {
			System.out.println(line);
		}

		br.close();
		in.close();
		fs.close();

	}

	/**
	 * 读取hdfs中文件的指定偏移量范围的内容
	 * 
	 * 
	 * 作业题：用本例中的知识，实现读取一个文本文件中的指定BLOCK块中的所有数据
	 * 
	 * @throws IOException
	 * @throws IllegalArgumentException
	 */
	@Test
	public void testRandomReadData() throws IllegalArgumentException, IOException {

		FSDataInputStream in = fs.open(new Path("/xx.dat"));

		// 将读取的起始位置进行指定
		in.seek(12);

		// 读16个字节
		byte[] buf = new byte[16];
		in.read(buf);

		System.out.println(new String(buf));

		in.close();
		fs.close();

	}

	/**
	 * 往hdfs中的文件写内容
	 * 
	 * @throws IOException
	 * @throws IllegalArgumentException
	 */

	@Test
	public void testWriteData() throws IllegalArgumentException, IOException {

		FSDataOutputStream out = fs.create(new Path("/zz.jpg"), false);

		// D:\images\006l0mbogy1fhehjb6ikoj30ku0ku76b.jpg

		FileInputStream in = new FileInputStream("D:/images/006l0mbogy1fhehjb6ikoj30ku0ku76b.jpg");

		byte[] buf = new byte[1024];
		int read = 0;
		while ((read = in.read(buf)) != -1) {
			out.write(buf,0,read);
		}
		
		in.close();
		out.close();
		fs.close();

	}

}