大数据学习（五）：java操作HDFS

设置IP映射

在C:\Windows\System32\drivers\etc\hosts中配置虚拟机相关映射

192.168.44.128		spark1
192.168.44.129		spark2
192.168.44.130		spark3

代码

package com.bigdata.hdfs;

import java.io.IOException;
import java.net.URI;
import java.util.Arrays;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

public class HDFSClientDemo {
	
	
	FileSystem fs = null;
	
	/**
	 * 初始化
	 * @throws Exception
	 */
	@Before
	public void init() throws Exception{
		Configuration conf = new Configuration();
		//设置副本数量
		conf.set("dfs.replication", "2");
		//设置块大小
		conf.set("dfs.blocksize", "64m");
		
		fs = FileSystem.get(new URI("hdfs://spark1:9000/"), conf, "root");
		
	}
	
	/**
	 * 将本地文件上传至HDFS中
	 * @throws IOException 
	 * @throws IllegalArgumentException 
	 */
	@Test
	public void testPut() throws IllegalArgumentException, IOException {
		// 上传一个文件到HDFS中
		fs.copyFromLocalFile(new Path("D:/PDF/4.pdf"), new Path("/aaa/"));
		fs.close();
	}
	
	
	/**
	 * 从HDFS中下载文件到客户端本地磁盘
	 * @throws IOException 
	 * @throws IllegalArgumentException 
	 */
	@Test
	public void testGet() throws IllegalArgumentException, IOException{
		
		fs.copyToLocalFile(new Path("/aaa/1.pdf"), new Path("D:/xxx.pdf"));
		fs.close();
		
	}
	
	
	/**
	 * 在hdfs内部移动文件\修改名称
	 */
	@Test
	public void testRename() throws Exception{
		
		fs.rename(new Path("/aaa/4.pdf"), new Path("/aaa/3.pdf"));
		
		fs.close();
		
	}
	
	/**
	 * 在HDFS中创建文件夹
	 */
	@Test
	public void testMkdirs() throws Exception{
		
		fs.mkdirs(new Path("/xx/yy/zz"));
		
		fs.close();
	}
	
	
	/**
	 * HDFS中删除文件或文件夹
	 */
	@Test
	public void testDelete() throws Exception{
		
		fs.delete(new Path("/xx"), true);
		
		fs.close();
	}
	
	
	
	/**
	 * 查询HDFS指定目录下的文件信息
	 */
	@Test
	public void testLs() throws Exception{
		// 只查询文件的信息,不返回文件夹的信息
		RemoteIterator<LocatedFileStatus> iter = fs.listFiles(new Path("/"), true);
		
		while(iter.hasNext()){
			LocatedFileStatus status = iter.next();
			System.out.println("文件全路径："+status.getPath());
			System.out.println("块大小："+status.getBlockSize());
			System.out.println("文件长度："+status.getLen());
			System.out.println("副本数量："+status.getReplication());
			System.out.println("块信息："+Arrays.toString(status.getBlockLocations()));
			
			System.out.println("--------------------------------");
		}
		fs.close();
	}
	
	/**
	 * 查询HDFS指定目录下的文件和文件夹信息
	 */
	@Test
	public void testLs2() throws Exception{
		FileStatus[] listStatus = fs.listStatus(new Path("/"));
		
		for(FileStatus status:listStatus){
			System.out.println("文件全路径："+status.getPath());
			System.out.println(status.isDirectory()?"这是文件夹":"这是文件");
			System.out.println("块大小："+status.getBlockSize());
			System.out.println("文件长度："+status.getLen());
			System.out.println("副本数量："+status.getReplication());
			
			System.out.println("--------------------------------");
		}
		fs.close();
	}
	
	/**
	 * 读取HDFS文件中内容
	 * @throws IOException 
	 * @throws IllegalArgumentException 
	 */
	@Test
	public void readHDFSData() throws Exception {
		FSDataInputStream in = fs.open(new Path("/aaa/readme.txt"));
		BufferedReader br = new BufferedReader(new InputStreamReader(in));
		String line = null;
		while((line=br.readLine())!=null) {
			System.out.println(line);
		}
		br.close();
		in.close();
		fs.close();
	}
	
	
	/**
	 * 读取hdfs中文件的指定偏移量范围的内容
	 * @throws IOException
	 * @throws IllegalArgumentException
	 */
	@Test
	public void testRandomReadHDFSData() throws IllegalArgumentException, IOException {

		FSDataInputStream in = fs.open(new Path("/aaa/readme1.txt"));

		// 将读取的起始位置进行指定
		in.seek(13);

		// 读12个字节
		byte[] buf = new byte[12];
		in.read(buf);

		System.out.println(new String(buf));

		in.close();
		fs.close();

	}
	/**
	 * 向HDFS写数据 
	 * @throws IOException 
	 * @throws IllegalArgumentException 
	 */
	@Test
	public void writeData2HDFS() throws IllegalArgumentException, IOException {
		FSDataOutputStream  out = fs.create(new Path("/aaa/1.jpg"), true);
		InputStream in = new FileInputStream("D://timg.jpg");
		byte[] buf = new byte[1024];
		int read = 0;
		while((read=in.read(buf))!=-1) {
			out.write(buf, 0, read);
		}
		in.close();
		out.close();
		fs.close();
	}
}