大数据学习4

hdfs javaApi实战：

maven：

<dependency>
  <groupId>org.apache.hadoop</groupId>
  <artifactId>hadoop-client</artifactId>
  <version>2.6.0</version>
</dependency>

 

package com.lc.hadoop.hdfsAPP;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.net.URI;

/**
 * hadoop hdfs 测试
 */
public class hdfsApp {
    private FileSystem fileSystem = null;
    private Configuration configuration = null;
    private String hadoopPath = "hdfs://www.imooc.com:8020";

//    创建目录
    @Test
    public void mkdir() throws Exception{
        fileSystem.mkdirs(new Path("/lcnewpath"));
        System.out.println("创建文件夹");
    }
//    创建文件
    @Test
    public void create() throws Exception{
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("lcnewpath/ha"), new Progressable() {
            @Override
            public void progress() {
                //带进度信息
                System.out.println(".");
            }
        });
        fsDataOutputStream.write("hello ,hadoop".getBytes());
        System.out.println("创建文件");;
    }
//    查看文件的内容
    @Test
    public void cat() throws Exception{
        FSDataInputStream fsDataInputStream = fileSystem.open(new Path("lcnewpath/ha"));
        IOUtils.copyBytes(fsDataInputStream,System.out,1024);
//        fsDataInputStream.close();

        BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream("lcc.txt"));
        byte[] bytes = new byte[1024];
        int lenth = fsDataInputStream.read(bytes);
        while (lenth != -1){
            bufferedOutputStream.write(bytes,0,lenth);
            lenth = fsDataInputStream.read(bytes);
        }
        bufferedOutputStream.close();
    }
//    重命名
    @Test
    public void reName() throws Exception{
        Path prePath = new Path("/lcnewpath");
        Path nowPath = new Path("lcnowpath");
        boolean bo = fileSystem.rename(prePath,nowPath);
        System.out.println(bo);
    }
//    从本地拷贝到hdfs
    @Test
    public void copyFromLocal() throws Exception{
        fileSystem.copyFromLocalFile(
                new Path(new File("lcc.txt").toString()),
                new Path("/lc"));
    }
//    从hdfs文件系统上下载文件
    @Test
    public void copyToLocalFile() throws Exception{
        fileSystem.copyFromLocalFile(
                new Path("hdfsPath")
                ,new Path("localPath")
        );
    }
//    显示某个文件夹下的文件数
    @Test
    public void list() throws Exception{
        FileStatus[]  fileStatus = fileSystem.listStatus(
                new Path("/user"));
        for(FileStatus fileStatus1 : fileStatus){
            System.out.println(fileStatus1.isDirectory());
            System.out.println(fileStatus1.getBlockSize());
            System.out.println(fileStatus1.getReplication());
        }
    }
    @Before
    public void setup() throws Exception{
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(hadoopPath),configuration,"lc");
    }

    @After
    public void tearDown() throws Exception{
        configuration = null;
        fileSystem = null;
    }
}