一、一般操作
package hdfs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.Iterator;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;
/**
*
* 客户端去操作hdfs时,是有一个用户身份的
* 默认情况下,hdfs客户端api会从jvm中获取一个参数来作为自己的用户身份:-DHADOOP_USER_NAME=hadoop
* 也可以在构造客户端fs对象时,通过参数传递进去
* @author
*
*/
public class HdfsClientDemo {
FileSystem fs = null;
Configuration conf = null;
@Before
public void init() throws Exception{
conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://vm1:9000");
conf.set("dfs.replication", "5");
//拿到一个文件系统操作的客户端实例对象
fs = FileSystem.get(conf);
//可以直接传入 uri和用户身份
fs = FileSystem.get(new URI("hdfs://vm1:9000"),conf,"hadoop");
}
/**
* 上传文件
* @throws Exception
*/
@Test
public void testUpload() throws Exception {
fs.copyFromLocalFile(new Path("c:/aa.txt"), new Path("/aa.txt"));
fs.close();
}
/**
* 下载文件
* @throws Exception
*/
@Test
public void testDownload() throws Exception {
fs.copyToLocalFile(new Path("/aa.txt"), new Path("d:/"));
fs.close();
}
/**
* 创建目录
* @throws Exception
*/
@Test
public void testMkdir() throws Exception {
boolean mkdirs = fs.mkdirs(new Path("/testmkdir/aaa/bbb"));
System.out.println(mkdirs);
}
/**
* 删除目录
* @throws Exception
*/
@Test
public void testDelete() throws Exception {
boolean flag = fs.delete(new Path("/testmkdir/aaa"), true);
System.out.println(flag);
}
/**
* 递归所有文件
* @throws Exception
*/
@Test
public void testLs() throws Exception {
RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
while(listFiles.hasNext()){
LocatedFileStatus fileStatus = listFiles.next();
System.out.println("blocksize: " +fileStatus.getBlockSize());
System.out.println("owner: " +fileStatus.getOwner());
System.out.println("Replication: " +fileStatus.getReplication());
System.out.println("Permission: " +fileStatus.getPermission());
System.out.println("Name: " +fileStatus.getPath().getName());
System.out.println("------------------");
BlockLocation[] blockLocations = fileStatus.getBlockLocations();
for(BlockLocation b:blockLocations){
System.out.println("块起始偏移量: " +b.getOffset());
System.out.println("块长度:" + b.getLength());
//块所在的datanode节点
String[] datanodes = b.getHosts();
for(String dn:datanodes){
System.out.println("datanode:" + dn);
}
}
}
}
}
二、流式操作
package hdfs;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Test;
/**
* 用流的方式来操作hdfs上的文件
* 可以实现读取指定偏移量范围的数据
* @author
*
*/
public class HdfsStreamAccess {
FileSystem fs = null;
Configuration conf = null;
@Before
public void init() throws Exception{
conf = new Configuration();
//可以直接传入 uri和用户身份
fs = FileSystem.get(new URI("hdfs://mini1:9000"),conf,"hadoop");
}
/**
* 通过流的方式上传文件到hdfs
* @throws Exception
*/
@Test
public void testUpload() throws Exception {
FSDataOutputStream outputStream = fs.create(new Path("/cc.txt"), true);
FileInputStream inputStream = new FileInputStream("c:/cc.txt");
IOUtils.copy(inputStream, outputStream);
}
/**
* 通过流的方式获取hdfs上数据
* @throws Exception
*/
@Test
public void testDownLoad() throws Exception {
FSDataInputStream inputStream = fs.open(new Path("/cc.txt"));
FileOutputStream outputStream = new FileOutputStream("d:/cc.txt");
IOUtils.copy(inputStream, outputStream);
}
/**
* 指定偏移量
* @throws Exception
*/
@Test
public void testRandomAccess() throws Exception{
FSDataInputStream inputStream = fs.open(new Path("/cc.txt"));
inputStream.seek(12);
FileOutputStream outputStream = new FileOutputStream("d:/cc2.txt");
IOUtils.copy(inputStream, outputStream);
}
/**
* 显示hdfs上文件的内容
* @throws IOException
* @throws IllegalArgumentException
*/
@Test
public void testCat() throws IllegalArgumentException, IOException{
FSDataInputStream in = fs.open(new Path("/cc.txt"));
IOUtils.copy(in, System.out);
// IOUtils.copyBytes(in, System.out, 1024);
}
}