hdfs操作

1.新建java工程。xia

2.导包。导入/share/hadoop/hdfs 下的hadoop-hdfs-2.7.7.jar包, /share/hadoop/hdfs/lib 下的所有包， /share/hadoop/common 下的hadoop-common-2.7.7.jar包 ，  /share/hadoop/common/lib的所有包。

package week01;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

public class HdfsUtil {

	/**
	 * 上传文件  比较底层的写法
	 * @throws IOException 
	 */
	/*@Test
	public void upload() throws IOException{
		//拿到hdfs的客户端
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://week01:9000");
		
		FileSystem fs = FileSystem.get(conf);
		
		
		//上传的地方
		Path  dst=new Path("hdfs://week01:9000/aa/test.txt");
		
		FSDataOutputStream os = fs.create(dst);
		
		//本地文件路径
		FileInputStream is = new FileInputStream("c:/test.txt");
		
		IOUtils.copy(is, os);
	}*/
	
	
	FileSystem fs = null;
	
	@Before
	public void init() throws IOException, InterruptedException, URISyntaxException{
		
		//读取classpath中的XXX-site.xml配置文件，并解析内容，封装到conf对象中
		Configuration conf = new Configuration();
		
		//可以手动修改配置文件中的配置信息，会覆盖原配置文件中的信息
		conf.set("fs.defaultFS", "hdfs://week01:9000");
		
		//根据配置信息，获取一个具体文件系统的客户端操作实例对象
		//fs = FileSystem.get(conf);
//		fs = FileSystem.get(new URI("hdfs://week01:9000"), conf, "hadoop");

	}
	
	/**
	 * 使用封装好的方法  
	 * @throws IllegalArgumentException
	 * @throws IOException
	 */
	@Test
	public void upload2() throws IllegalArgumentException, IOException{
		fs.copyFromLocalFile(new Path("c:/test.txt"), new Path("hdfs://week01:9000/aaa/bb/cc"));
		
	}
	
	/**
	 * 下载文件  使用封装好的方法  
	 * @throws Exception 
	 * @throws IllegalArgumentException 
	 */
	@Test
	public void download() throws Exception{
		fs.copyToLocalFile(new Path("hdfs://week01:9000/aa/test2.txt"), new Path("c:/test2.txt"));
	}
	
	/**
	 * 查看文件信息
	 * @throws Exception 
	 * @throws IllegalArgumentException 
	 * @throws FileNotFoundException 
	 * 
	 */
	@Test
	public void listFiles() throws FileNotFoundException, IllegalArgumentException, Exception{
		//listFiles 列出的是文件信息，而且提供递归遍历
		RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);
		
		while(files.hasNext()){
			LocatedFileStatus file = files.next();
			Path filepath = file.getPath();
			String fileName = filepath.getName();
			System.out.println(fileName);
		}
	}
	
	/**
	 * 查看文件夹
	 * @throws IOException 
	 * @throws IllegalArgumentException 
	 * @throws FileNotFoundException 
	 */
	@Test
	public void listJ() throws FileNotFoundException, IllegalArgumentException, IOException{
		//listStatus 方法可以列出文件和文件夹的信息，但是不提供自带的递归遍历
		FileStatus[] listStatus = fs.listStatus(new Path("/"));
		
		for (FileStatus status : listStatus) {
			String name = status.getPath().getName();
			System.out.println(name + (status.isDirectory()? "is dir":"is file"));
		}
		
	}

	/**
	 * 创建文件夹、目录
	 * @throws IOException 
	 * @throws IllegalArgumentException 
	 */
	@Test
	public void mkdir() throws IllegalArgumentException, IOException{
		fs.mkdirs(new Path("/aaa/bb/cc"));
		
	}
	/**
	 * 删除文件或文件夹
	 * @throws IOException 
	 * @throws IllegalArgumentException 
	 */
	@Test
	public void delete() throws IllegalArgumentException, IOException{
		fs.delete(new Path("/aa"), true);
	}
	
	/**
	 * 
	 */
}

FileSystem的调用。