hdfs基本操作

@Test
public void test1() {
    System.out.println(System.getenv("HADOOP_HOME"));
}


@Test
//创建文件夹
public void listFiles() throws IOException {
    Configuration conf = new Configuration();
    conf.set("fs.default.name", "hdfs://ip);
    FileSystem fs = null;
    fs = FileSystem.get(conf);
    boolean result = fs.mkdirs(new Path("/tt1"));
    System.out.println(result);
    fs.close();
}

@Test
//读取文件
public void reatTxt() throws IOException {
    String uri = "hdfs://ip/test/hdfs/a.txt";
    Configuration cfg = new Configuration();
    FileSystem fs = FileSystem.get(URI.create(uri), cfg);
    InputStream in = null;
    try {
        in = fs.open(new Path(uri));
        IOUtils.copyBytes(in, System.out, 4096, false);
    } catch (Exception e) {
        System.out.println(e.getMessage());
    } finally {
        IOUtils.closeStream(in);
    }
}


@Test
//上传文件
public void putFile() throws IOException {
    //本地文件路径
    String local = "G:/hdfs/a.txt";
    String dest = "hdfs://ip/test/a.txt";
    InputStream in = new BufferedInputStream(new FileInputStream(local));
    Configuration cfg = new Configuration();
    FileSystem fs = FileSystem.get(URI.create(dest), cfg);
    OutputStream out = fs.create(new Path(dest));
    IOUtils.copyBytes(in, out, 4096, true);
    fs.close();
    IOUtils.closeStream(in);
}

@Test
//上传本地目录
public void putFiles() throws IOException {
    String hdfsPath = "hdfs://ip/user1";
    String localPath = "G:/hdfs";
    Configuration conf = new Configuration();
    try {
        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);
        Path hdfs_path = new Path(hdfsPath);
        Path local_path = new Path(localPath);
        fs.copyFromLocalFile(local_path, hdfs_path);
        fs.close();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

@Test
//下载文件
public void getFile() throws IOException {
    String hdfsPath = "hdfs://ip/test/test.txt";
    String localPath = "G:/hdfs";
    Configuration conf = new Configuration();
    conf.set("hadoop.home.dir", "G:/hadoop-3.0.3");
    FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);
    Path hdfs_path = new Path(hdfsPath);
    Path local_path = new Path(localPath);
    fs.copyToLocalFile(hdfs_path, local_path);
    fs.close();
}

@Test
//下载目录
public void getFiles() throws IOException {
    String hdfsPath = "hdfs://ip/test";
    String localPath = "G:/hdfs";
    Configuration conf = new Configuration();
    conf.set("hadoop.home.dir", "G:/hadoop-3.0.3");
    FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);
    Path hdfs_path = new Path(hdfsPath);
    Path local_path = new Path(localPath);
    fs.copyToLocalFile(hdfs_path, local_path);
    fs.close();
}

@Test
//删除文件
public void deleteFile() throws IOException {
    String uri = "hdfs://ip/user1";
    Path path = new Path(uri);
    Configuration conf = new Configuration();
    try {
        FileSystem fs = path.getFileSystem(conf);
        //递归删除文件夹及文件夹下的文件
        boolean b = fs.delete(path, true);
        System.out.println(b);
        fs.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}

@Test
//输出HDFS指定目录下的文件和子目录
public void run1() throws IOException {
    String uri = "hdfs://ip/tt1";
    Configuration cfg = new Configuration();
    FileSystem fs = FileSystem.get(URI.create(uri), cfg);
    Path path = new Path(uri);
    FileStatus[] fss = fs.listStatus(path);
    for (FileStatus f : fss) {
        if (f.isFile())
            System.out.println("File:" + f.getPath().toString());
        else
            System.out.println("Dir:" + f.getPath().toString());
    }
}