HDFS-IO流方式实现文件上传下载

1.IO流实现完整文件上传下载

//使用IO流上传文件

@Test
public void putFileToHDFS() throws IOException, InterruptedException, URISyntaxException {
	Configuration configuration = new Configuration();
	//获取文件系统
	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.33.17:9000"), 
			configuration, "hadoop");
	//输入流：本地文件
	FileInputStream fis=new FileInputStream(new File("e://hadoop/hello.txt")); 
	//输出流：HDFS路径
	FSDataOutputStream fos = fs.create(new Path("/0423/hello.txt"));
	
	//io对拷
	IOUtils.copyBytes(fis, fos, configuration);
	
	//关闭流
	IOUtils.closeStream(fis);
	IOUtils.closeStream(fos);
}

/*
 * 使用IO流下载文件
 * */

@Test
public void downFileFromHDFS() throws IOException, InterruptedException, URISyntaxException {
	Configuration configuration = new Configuration();
	//获取文件系统
	FileSystem fs = FileSystem.get(new URI("hdfs://192.168.33.17:9000"), 
			configuration, "hadoop");
	
	//获取输入流
	FSDataInputStream fis = fs.open(new Path
			("/0423/hello.txt"));
	
	//拷贝到控制台输出
	IOUtils.copyBytes(fis, System.out, configuration);
	
	//关闭流
	IOUtils.closeStream(fis);
}

2.IO流实现文件分片下载

//读取第一部分

@Test
	public void downFilePart1() throws IOException, InterruptedException, URISyntaxException {
		Configuration configuration = new Configuration();
		//获取文件系统
		FileSystem fs = FileSystem.get(new URI("hdfs://192.168.33.17:9000"), 
				configuration, "hadoop");
		
		//获取输入流
		FSDataInputStream fis = fs.open(new Path("/test/hadoop-2.6.4.tar.gz"));
		
		//获取输出流
		FileOutputStream fos = new FileOutputStream
				(new File("e://hadoop/hadoop-2.6.4.tar.gz_part1"));
		
		//流对拷
		byte[] buff=new byte[1024];
		for(int i=0;i<1024*128;i++) {
			fis.read(buff);
			fos.write(buff);
		}
		
		//关闭流
		IOUtils.closeStream(fis);
		IOUtils.closeStream(fos);
		fs.close();
	}

//读取第二部分

@Test
	public void downFilePart2() throws IOException, InterruptedException, URISyntaxException {
		Configuration configuration = new Configuration();
		//获取文件系统
		FileSystem fs = FileSystem.get(new URI("hdfs://192.168.33.17:9000"), 
				configuration, "hadoop");
		
		//获取输入流
		FSDataInputStream fis = fs.open(new Path("/test/hadoop-2.6.4.tar.gz"));
		fis.seek(1024*1024*128);
		//获取输出流
		FileOutputStream fos=new FileOutputStream(new File("e://hadoop/hadoop-2.6.4.tar.gz_part2"));
		
		//流对拷
		IOUtils.copyBytes(fis, fos, configuration);
		
		//关闭流
		IOUtils.closeStream(fis);
		IOUtils.closeStream(fos);
		
		fs.close();
	}