使用IDEA编写客户端程序操作HDFS

以API方式编写程序操作HDFS

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;


public class HdfsClientDemo {
    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration(); //创建配置信息对象
        configuration.set("fs.defaultFS","hdfs://wdp80:8020"); //设置配置信息
        //FileSystem fs = FileSystem.get(new URI("hdfs://wdp80:8020"),configuration,"bduser");直接配置访问集群的路径和访问集群的用户名
        FileSystem fileSystem = FileSystem.get(configuration); //此处有IOException，为了方便查看代码，直接main方法抛出Exception
        fileSystem.copyFromLocalFile(new Path("d:/foo.txt"),new Path("/foo.txt"));//向hdfs上传文件
        fileSystem.copyToLocalFile(new Path("/foo.txt"),new Path("d:/foo.txt"));//从hdfs下载文件
        fileSystem.mkdirs(new Path("hdfs://wdp80:8020/user/haha"));  //创建目录
        fileSystem.rename(new Path("hdfs://wdp80:8020/user/haha"),new Path("hdfs://wdp80:8020/user/xixi"));//文件名更改
        fileSystem.delete(new Path("hdfs://wdp80:8020/user/haha"),true);  //删除文件或文件夹
        // 文件详情查看
        RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"),true);
        while (listFiles.hasNext()){
            LocatedFileStatus fileStatus = listFiles.next();
            System.out.println(fileStatus.getPath().getName());
            System.out.println(fileStatus.getBlockSize());
            System.out.println(fileStatus.getPermission());
            System.out.println(fileStatus.getLen());
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            for(BlockLocation bl :blockLocations){
                System.out.println("block-offset:"+bl.getOffset());
                String[] hosts = bl.getHosts();
                for(String host:hosts){
                    System.out.println(host);
                }
                System.out.println("------分割线-------");
            }
        }
        // 查看文件夹信息
        FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
        for (FileStatus status : listStatus){
            if (status.isFile()){
                System.out.println("f--"+status.getPath().getName());
            }else {
                System.out.println("d--"+status.getPath().getName());
            }
        }
        fileSystem.close();
        System.out.println("over");
    }
}

以IO的方式编写程序操作HDFS

文件上传：

import org.apache.hadoop.io.IOUtils;
import java.io.File;
import java.io.FileInputStream;
import java.net.URI;

public void putFileToHDFS() throws Exception{  // 文件上传
    // 1 创建配置信息对象
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(new URI("hdfs://wdp80:8020"),configuration, "bduser");
    // 2 创建输入流
    FileInputStream inStream = new FileInputStream(new File("d:/foo.txt"));
    // 3 获取输出路径
    String putFileName = "hdfs://wdp80:8020/foo.txt";
    Path writePath = new Path(putFileName);
    // 4 创建输出流
    FSDataOutputStream outStream = fs.create(writePath);
    // 5 流对接
    try{
        IOUtils.copyBytes(inStream, outStream, 4096, false);
    }catch(Exception e){
        e.printStackTrace();
    }finally{
        IOUtils.closeStream(inStream);
        IOUtils.closeStream(outStream);
    }
}