hadoop学习5-HDFS API学习

复习上一次课总结

1）HDFS读过程

DistributedFileSystem =>FSDataInputStream =>DFSClient.open（RPC通信机制)=>NN.open

2)HDFS写过程

DistributeFileSystem => FSDataOutputStream => DFSClient.create（RPC通信机制)=>NN.create

3)SecondaryNamenode的作用与机制

SNN不是完全意义上的NN的备份

拉取Fsimage和edits文件的SNN的内存中进行合并

fs.checkpoint.period

fs.checkpoint.size

fs.checkpoint.dir

hadoop2.x以及hadoop0.23后

checkpoint node 与secondaryNamenode完全一样

backup node 完全意义上的namenode备份

4）一旦丢失NN或者元数据信息，可以通过SNN检查点目录恢复元数据信息

hadoop namenode -importCheckpoint  导入检查点信息

hadoop-daemon.sh start namenode  启动namenode

5)机架感知

默认情况下所有DN被认为是出于同一个机架上，不管是否物理上属于同一个机架

/default-rack

topology.script.file.name属性值是一个脚本文件可以是python或shell，这个脚本里面写的是真正意义上的网络拓扑结构图

/d1/rack1/dn1 

HDFS API

地址:hadoop.apache.org/docs/current1/api

下面通过一些testcase来展示HDFS API部分代码

public static String hdfsUrl = "hdfs://192.168.1.201:9100";
	
	//create HDFS folder
	@Test
	public void testHDFSMKdir() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
		Path path = new Path("/test");
		fs.mkdirs(path);		
	}
	
	//create HDFS file
	@Test
	public void testCreateFile() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(URI.create(hdfsUrl), conf);
		Path path = new Path("/test/a.txt");
		FSDataOutputStream out = fs.create(path);
		out.write("hello hadoop!".getBytes());
	}
	
	//rename a file name
	@Test
	public void testRenameFile() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(URI.create(hdfsUrl), conf);
		Path path = new Path("/test/a.txt");
		Path newPath = new Path("/test/b.txt");
		System.out.print(fs.rename(path, newPath));
	}
	
	//upload a local file
	@Test
	public void testUploadLocalFile1() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs =FileSystem.get(URI.create(hdfsUrl), conf);
		Path src = new Path("/home/hadoop-1.2.1/bin/rcc");
		Path dst = new Path("/test");
		fs.copyFromLocalFile(src, dst);
	}
	
	//other way upload a local file
		@Test
		public void testUploadLocalFile2() throws IOException{
			Configuration conf = new Configuration();
			FileSystem fs =FileSystem.get(URI.create(hdfsUrl), conf);
			InputStream in = new BufferedInputStream(new FileInputStream(new File("/home/hadoop-1.2.1/bin/rcc")));
			FSDataOutputStream out = fs.create(new Path("/test/rcc"));
			IOUtils.copyBytes(in, out, 1024);
		}
	
	//other way upload a local file
	@Test
	public void testUploadLocalFile3() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs =FileSystem.get(URI.create(hdfsUrl), conf);
		InputStream in = new BufferedInputStream(new FileInputStream(new File("/home/hadoop-1.2.1/bin/rcc")));
		FSDataOutputStream out = fs.create(new Path("/test/data"),new Progressable(){
			
			@Override
			public void progress(){
				System.out.println(".");
			}
		});
		IOUtils.copyBytes(in, out, 1024);
	}
	
	//List files under folder
	@Test
	public void testListFiles() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
		Path dst = new Path("/test");
		FileStatus[] files = fs.listStatus(dst);
		for(FileStatus file:files){
			System.out.println(file.getPath().toString());
		}
	}
	
	//List block under folder
		@Test
		public void testGetBlockInfo() throws IOException{
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
			Path dst = new Path("/test/data");
			FileStatus fileStatus = fs.getFileStatus(dst);
			BlockLocation[] blkLoc = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
			
			for(BlockLocation loc:blkLoc){
				for(int i = 0;i<loc.getHosts().length;i++){
					System.out.println(loc.getHosts()[i]);
				}
			}
		}