通过IO流操作HDFS

package com.atguigu.hdfs;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
import org.mortbay.util.IO;

/**通过IO流操作HDFS
 * @author huangyu
 *
 */
public class IOToHdfs {
	@Test
	// 文件的上传 从本地通过IO流的方式上传到HDFS服务器
	public void putFileToHDFS() throws IOException, InterruptedException,
			URISyntaxException {
		Configuration conf = new Configuration();
		// 1,获取文件系统
		FileSystem fileSystem = FileSystem.get(
				new URI("hdfs://hadoop103:9000"), conf, "atguigu");
		// 2,获取输入流
		FileInputStream fsInputStream = new FileInputStream(new File(
				"e:/hadoop-2.7.2.tar.gz"));
		// 3,获取输出流
		FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path(
				"/user/atguigu/hadoop-2.7.2.tar.gz"));
		// 4,流的拷贝
		try {
			IOUtils.copyBytes(fsInputStream, fsDataOutputStream, conf);
		} catch (Exception e) {
			// TODO: handle exception
		} finally {
			// 5,关闭资源
			IOUtils.closeStream(fsInputStream);
			IOUtils.closeStream(fsDataOutputStream);
		}

	}
	@Test
	// 文件的下载
	public void getFileFromHDFS() throws IOException, InterruptedException,
			URISyntaxException {
		Configuration conf = new Configuration();
		// 1,获取文件系统
		FileSystem fileSystem = FileSystem.get(
				new URI("hdfs://hadoop103:9000"), conf, "atguigu");
		// 2,获取输入流
		FSDataInputStream fsDataInputStream = fileSystem.open(new Path(
				"/user/atguigu/test/a.txt"));
		// 3,获取输出流
		FileOutputStream fos = new FileOutputStream(new File("e:/huangyu.txt"));

		
		// 5,关闭资源
		try {
			// 4,流的拷贝
			IOUtils.copyBytes(fsDataInputStream, fos, conf);
		} catch (Exception e) {
			// TODO: handle exception
		} finally {
			IOUtils.closeStream(fsDataInputStream);
			IOUtils.closeStream(fos);
		}

	}
	@Test
	//下载大文件的第一块数据
	public void getFileFromHDFSSeek1() throws IOException, InterruptedException, URISyntaxException{
		Configuration conf = new Configuration();
		//1,获取文件系统
		FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop103:9000"), conf, "atguigu");
		//2,获取输入流
		FSDataInputStream fsDataInputStream = fileSystem.open(new Path("/user/atguigu/hadoop-2.7.2.tar.gz"));
		//3,获取输出流
		FileOutputStream fos = new FileOutputStream(new File("e:/1/hadoop-2.7.2.tar.gz.part1"));
		
		
		
		try {
			//4,流的拷贝	
			byte[] buf = new byte[1024 * 1024];
			
			for(int i = 0;i < 128;i++){
				fsDataInputStream.read(buf);
				fos.write(buf);
			}
		} catch (Exception e) {
			// TODO: handle exception
		}finally{
			IOUtils.closeStream(fsDataInputStream);
			IOUtils.closeStream(fos);
		}
	}
	@Test
	//下载大文件的第二块数据
	public void getFileFromHDFSSeek2() throws IOException, InterruptedException, URISyntaxException{
		Configuration conf = new Configuration();
		//1,获取文件系统
		FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop103:9000"), conf, "atguigu");
		//2,获取输入流
		FSDataInputStream fsDataInputStream = fileSystem.open(new Path("/user/atguigu/hadoop-2.7.2.tar.gz"));
		//3,获取输出流
		FileOutputStream fos = new FileOutputStream(new File("e:/1/hadoop-2.7.2.tar.gz.part2"));
		//4，流的对接，定位到128M
		fsDataInputStream.seek(1024*1024*128);
		
		//6，关闭资源
		try {
			//5,流的对接
			IOUtils.copyBytes(fsDataInputStream, fos, conf);
		} catch (Exception e) {
			// TODO: handle exception
		}finally{
			IOUtils.closeStream(fsDataInputStream);
			IOUtils.closeStream(fos);
		}
		
	}
}