HDFS的API学习

package com.jt;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class Starter {

public static void main(String[] args) throws Exception {
	testConnectNamenode();

// testPut();

// testDelete();
// testLs1();
// testGetBlock();
}

//连接name节点并读取数据
public static void testConnectNamenode() throws Exception {
	//获取hadoop的环境变量信息，可以通过此对象设置环境，通过代码设定的优先级要高于文件设置的优先级
	//代码设定的生效范围是局部的
	Configuration conf=new Configuration(); 

// conf.set(“hadoop.home.dir”, “/Users/joe/Documents/software/hadoop-2.7.1”);
System.setProperty(“hadoop.home.dir”, “/Users/joe/Documents/software/hadoop-2.7.1”);
//得到文件输入流

	FileSystem  fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"), conf);
	
	InputStream in=fs.open(new Path("/ex/1.txt"));
	
	OutputStream out=new FileOutputStream("1.txt");
	
	IOUtils.copyBytes(in, out, conf);
	
	out.close();
	in.close();
	fs.close();
	
	
}

//上传文件到hdfs上
public static void  testPut() throws Exception{
	Configuration conf=new Configuration();
	
	conf.set("dfs.replication","1");
	
	System.setProperty("hadoop.home.dir", "/Users/joe/Documents/software/hadoop-2.7.1");
	
	FileSystem fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"),conf,"root");

// ByteArrayInputStream in=new ByteArrayInputStream(“hello hdfs”.getBytes());
InputStream in=new FileInputStream(new File("/Users/joe/Downloads/jtday15pm.mp4"));

	OutputStream  out=fs.create(new Path("/park02/jtday15pm.mp4"));
	IOUtils.copyBytes(in, out, conf);
	
	in.close();
	out.close();
	fs.close();
	
	
	
}

//从hdfs上删除文件
public static void testDelete() throws Exception{
	Configuration conf=new Configuration();
	System.setProperty("hadoop.home.dir", "/Users/joe/Documents/software/hadoop-2.7.1");
	
	FileSystem fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"), conf);
	
   fs.deleteOnExit(new Path("/park02/jtday15pm.mp4"));
   
   fs.close();
    
			
	
}

//查询指定目录下文件

public static void testLs() throws Exception{
	Configuration conf=new Configuration();
	System.setProperty("hadoop.home.dir","/Users/joe/Documents/software/hadoop-2.7.1");
	
	FileSystem fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"),conf);
	
	FileStatus[] listStatus = fs.listStatus(new Path("/"));
	
	for(FileStatus status:listStatus){
		System.out.println(status);
	}
	fs.close();
}

//递归查询

public static void testLs1() throws Exception{
	Configuration conf=new Configuration();
	System.setProperty("hadoop.home.dir","/Users/joe/Documents/software/hadoop-2.7.1");
	
	FileSystem fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"),conf);
	
    RemoteIterator<LocatedFileStatus> lf = fs.listFiles(new Path("/"),true);
	
	while (lf.hasNext()){
		System.err.println(lf.next());
	}
	
	fs.close();
	
}

//得到文件块信息

public static void testGetBlock() throws Exception{
	Configuration conf=new Configuration();
	System.setProperty("hadoop.home.dir","/Users/joe/Documents/software/hadoop-2.7.1");
	
	FileSystem fs=FileSystem.get(new URI("hdfs://10.211.55.8:9000"),conf);
	
   BlockLocation[] fbl = fs.getFileBlockLocations(new Path("/park02/jtday15pm.mp4"), 0, Integer.MAX_VALUE);
	for(BlockLocation bl:fbl){
		System.err.println(bl);
	}
   
   fs.close();
	
}

}