- package hdfs;
- import static org.junit.Assert.fail;
- import java.util.Arrays;
- import org.apache.hadoop.conf.Configuration;
- import org.apache.hadoop.fs.BlockLocation;
- import org.apache.hadoop.fs.FSDataOutputStream;
- import org.apache.hadoop.fs.FileStatus;
- import org.apache.hadoop.fs.FileSystem;
- import org.apache.hadoop.fs.Path;
- import org.apache.hadoop.hdfs.DistributedFileSystem;
- import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
- import org.junit.Test;
- public class TestHdfs {
- @Test
- public void test() {
- fail("Not yet implemented");
- }
- //上传本地文件到HDFS
- @Test
- public void testUpload() throws Exception{
- Configuration conf = new Configuration();
- conf.addResource(new Path("D:\\myeclipse\\Hadoop\\hadoopEx\\src\\conf\\hadoop.xml"));
- FileSystem hdfs = FileSystem.get(conf);
- Path src = new Path("F:\\lzp\\T.txt");
- Path dst = new Path("/");
- hdfs.copyFromLocalFile(src, dst);
- System.out.println("Upload to " + conf.get("fs.default.name"));
- FileStatus files[] = hdfs.listStatus(dst);
- for(FileStatus file : files){
- System.out.println(file.getPath());
- }
- }
- //创建HDFS文件
- @Test
- public void testCreate() throws Exception{
- Configuration conf = new Configuration();
- conf.addResource(new Path("D:\\myeclipse\\Hadoop\\hadoopEx\\src\\conf\\hadoop.xml"));
- byte[] buff = "hello world!".getBytes();
- FileSystem hdfs = FileSystem.get(conf);
- Path dst = new Path("/test");
- FSDataOutputStream outputStream = null;
- try{
- outputStream = hdfs.create(dst);
- outputStream.write(buff,0,buff.length);
- }catch(Exception e){
- e.printStackTrace();
- }finally{
- if(outputStream != null){
- outputStream.close();
- }
- }
- FileStatus files[] = hdfs.listStatus(dst);
- for(FileStatus file : files){
- System.out.println(file.getPath());
- }
- }
- //重命名HDFS文件
- @Test
- public void testRename() throws Exception{
- Configuration conf = new Configuration();
- conf.addResource(new Path("D:\\myeclipse\\Hadoop\\hadoopEx\\src\\conf\\hadoop.xml"));
- FileSystem hdfs = FileSystem.get(conf);
- Path dst = new Path("/");
- Path frpath = new Path("/test");
- Path topath = new Path("/test1");
- hdfs.rename(frpath, topath);
- FileStatus files[] = hdfs.listStatus(dst);
- for(FileStatus file : files){
- System.out.println(file.getPath());
- }
- }
- //刪除HDFS文件
- @Test
- public void testDel() throws Exception{
- Configuration conf = new Configuration();
- conf.addResource(new Path("D:\\myeclipse\\Hadoop\\hadoopEx\\src\\conf\\hadoop.xml"));
- FileSystem hdfs = FileSystem.get(conf);
- Path dst = new Path("/");
- Path topath = new Path("/test1");
- boolean ok = hdfs.delete(topath,false);
- System.out.println( ok ? "删除成功" : "删除失败");
- FileStatus files[] = hdfs.listStatus(dst);
- for(FileStatus file : files){
- System.out.println(file.getPath());
- }
- }
- //查看HDFS文件的最后修改时间
- @Test
- public void testgetModifyTime() throws Exception{
- Configuration conf = new Configuration();
- conf.addResource(new Path("D:\\myeclipse\\Hadoop\\hadoopEx\\src\\conf\\hadoop.xml"));
- FileSystem hdfs = FileSystem.get(conf);
- Path dst = new Path("/");
- FileStatus files[] = hdfs.listStatus(dst);
- for(FileStatus file : files){
- System.out.println(file.getPath() +"\t" + file.getModificationTime());
- }
- }
- //查看HDFS文件是否存在
- @Test
- public void testExists() throws Exception{
- Configuration conf = new Configuration();
- conf.addResource(new Path("D:\\myeclipse\\Hadoop\\hadoopEx\\src\\conf\\hadoop.xml"));
- FileSystem hdfs = FileSystem.get(conf);
- Path dst = new Path("/T.txt");
- boolean ok = hdfs.exists(dst);
- System.out.println( ok ? "文件存在" : "文件不存在");
- }
- //查看某个文件在HDFS集群的位置
- @Test
- public void testFileBlockLocation() throws Exception{
- Configuration conf = new Configuration();
- conf.addResource(new Path("D:\\myeclipse\\Hadoop\\hadoopEx\\src\\conf\\hadoop.xml"));
- FileSystem hdfs = FileSystem.get(conf);
- Path dst = new Path("/T.txt");
- FileStatus fileStatus = hdfs.getFileStatus(dst);
- BlockLocation[] blockLocations =hdfs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
- for(BlockLocation block : blockLocations){
- System.out.println(Arrays.toString(block.getHosts())+ "\t" + Arrays.toString(block.getNames()));
- }
- }
- //获取HDFS集群上所有节点名称
- @Test
- public void testGetHostName() throws Exception{
- Configuration conf = new Configuration();
- conf.addResource(new Path("D:\\myeclipse\\Hadoop\\hadoopEx\\src\\conf\\hadoop.xml"));
- DistributedFileSystem hdfs = (DistributedFileSystem)FileSystem.get(conf);
- DatanodeInfo[] dataNodeStats = hdfs.getDataNodeStats();
- for(DatanodeInfo dataNode : dataNodeStats){
- System.out.println(dataNode.getHostName() + "\t" + dataNode.getName());
- }
- }
- }
HDFS的JavaAPI操作
最新推荐文章于 2022-12-14 09:38:44 发布