HDFS的java操作

HDFS的客户端操作分为 Java和shell命令行

hdfs在生产应用中主要是客户端的开发，其核心步骤是从hdfs提供的api中构造一个HDFS的访问客户端对象，然后通过该客户端对象操作（增删改查）HDFS上的文件

1. 文件的增删改查

 package cn.yzx.bigdata.hdfs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map.Entry;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.DFSClient.Conf;
import org.junit.Before;
import org.junit.Test;

public class HdfsClientDemo {
	FileSystem fs=null;
	Configuration conf = null;
	@Before
	public void init() throws IOException {
	 conf = new Configuration();
	  conf.set("fs.defaultFS", "hdfs://hadoop01:9000");
	  conf.set("dfs.defaultFS", "5");
	  //拿到一个文件系统操作的客户端实例对象
		 fs = FileSystem.get(conf);
	
	}
	/*
	 * 上传文件
	 */
	
	@Test
	public void testUpload() throws Exception {
		fs.copyFromLocalFile(new Path("c:/AMTAG.BIN"), new Path("/AMTAG.BIN.copy"));
		fs.close();
	}
	/*
	 * 打印参数
	 */
	@Test
	public void testconf() {
		Iterator<Entry<String, String>> iterator = conf.iterator();
		while(iterator.hasNext()) {
			Entry<String, String> next = iterator.next();
			System.out.println(next.getKey()+" :  "+next.getValue());
		}
		
	}
	/*
	 * 创建文件夹
	 */
	@Test
	public void testMkdir() throws Exception {
		boolean mkdirs = fs.mkdirs(new Path("/testmkdirs/aaa"));
		System.out.println(mkdirs);
	}
	/*
	 * 删除
	 */
	@Test
	public void testdelete() throws Exception {
		boolean delete = fs.delete(new Path("/testmkdirs/aaa"), true);
		System.out.println(delete);
	}
	/*
	 * 查看 递归
	 */
	@Test
	public void testLs() throws Exception {
		RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);
		while(files.hasNext()) {
			LocatedFileStatus fileStatus = files.next();
			System.out.println("BlockSize   "+fileStatus.getBlockSize());
			System.out.println("Owner   "+fileStatus.getOwner());
			System.out.println("Replication   "+fileStatus.getReplication());
			System.out.println("Permission   "+fileStatus.getPermission());
			System.out.println("name   "+fileStatus.getPath().getName());
			BlockLocation[] blockLocations = fileStatus.getBlockLocations();
			for(BlockLocation blockLocation:blockLocations) {
				System.out.println("块的ID  "+blockLocation.getNames());
				System.out.println("块起始偏移量  "+blockLocation.getOffset());
				System.out.println("datanode "+blockLocation.getLength());
				String[] datanodes = blockLocation.getHosts();
				for(String s:datanodes) {
					System.out.println("datanode "+s);
				}
				
			}
			System.out.println("+++++++++++++++++++++++++++");
		}
	}
	/*
	 * 查看不用递归
	 */
	@Test
	public void testLs2() throws Exception {
		FileStatus[] listStatus = fs.listStatus(new Path("/"));
		for(FileStatus s:listStatus) {
			System.out.println("name "+s.getPath().getName());
			System.out.println(s.isFile()?"f":"dir");
		}
	}
	
}

2 。通过流的方式访问hdfs

hdfs支持随机定位进行文件读取，而且可以方便地读取指定长度

package cn.yzx.bigdata.hdfs;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Test;

/*
 * 用流的方式操作HDFS上的文件
 * 可以指定偏移量的范围的数据
 */
public class HdfsStreamAccess {
	FileSystem fs=null;
	Configuration conf = null;
	@Before
	public void init() throws IOException {
	 conf = new Configuration();
	  conf.set("fs.defaultFS", "hdfs://hadoop01:9000");
	  //拿到一个文件系统操作的客户端实例对象
		 fs = FileSystem.get(conf);
	
	}
	@Test
	public void testUpload() throws Exception {
		 //上传文件 anglebaby.love  true表示如果有就覆盖
		FSDataOutputStream outputstream = fs.create(new Path("/anglababy.txt"),true);
		FileInputStream inputStream = new FileInputStream("c:/anglababy.txt");
		IOUtils.copy(inputStream, outputstream);
	}
	/*
	 * 从指定位置开始读
	 */
	@Test
	public  void testRandAccess () throws IllegalArgumentException, IOException {
		FSDataInputStream inputStream = fs.open(new Path("/anglababy.txt"));
		inputStream.seek(12);
		FileOutputStream outputStream = new FileOutputStream("f:/anglababy.txt");
		IOUtils.copy(inputStream, outputStream);
		 
	}
}