hdfs api 操作

1、项目的管理主要用了maven，需要的依赖如下：

2、对hdfs操作代码如下：
private static FileSystem fs;
private static final String PROTOCOL = “hdfs”;
private static final String FLAG = “//”;
private static final String TERMINAL = “:”;

/**
 * 获得文件系统对象
 *
 * @param host
 * @param port
 * @return
 */
public static FileSystem getFs(String host, int port) {
    if (fs == null) {
        createFileSystem(host, port);
    }
    return fs;
}

/**
 * 创建一个HDFS文件系统对象
 *
 * @param host
 * @param port
 */
private static void createFileSystem(String host, int port) {
    String hdfs_url = PROTOCOL + TERMINAL + FLAG + host + FLAG + port;
    try {
        Configuration conf = new Configuration();
        fs = FileSystem.newInstance(URI.create(hdfs_url), conf);
    } catch (IOException e) {
        logger.error("创建文件系统对象失败," + e);
    }
}

/**
 * 获得一个文件输入流
 *
 * @param host
 * @param port
 * @param path
 * @return
 */
public static BufferedReader getHdfsReader(String host, int port, String path) {
    if (fs == null) {
        getFs(host, port);
    }
    Path filePath = new Path(path);
    BufferedReader reader = null;
    try {
        reader = new BufferedReader(new InputStreamReader(fs.open(filePath), "UTF-8"));
        fs.close();
    } catch (IOException e) {
        logger.error("获取文件输入流失败," + e);
    }
    return reader;
}

/**
 * 获取hdfs输出流
 *
 * @param host
 * @param port
 * @param dst
 */
public static FSDataOutputStream getHdfsWriter(String host, int port, String dst) {
    if (fs == null) {
        getFs(host, port);
    }
    Path dest = new Path(dst);
    FSDataOutputStream out = null;
    try {
        out = fs.create(dest);
        fs.close();
    } catch (IOException e) {
        logger.error("获取输出流失败," + e);
    }
    return out;
}

/**
 * 上传文件到hdfs
 * @param host
 * @param port
 * @param local
 * @param dst
 */
public static void uploadFileToHdfs(String host, int port, String local, String dst) {
    if(fs == null) {
        getFs(host, port);
    }
    Path locPath = new Path(local);
    Path destPath = new Path(dst);
    try {
        fs.copyFromLocalFile(false, locPath, destPath);
        fs.close();
    } catch (IOException e) {
        logger.error("上传文件失败,"+e);
    }
}

/**
 * 获取hdfs文件夹下的文件
 * @param host
 * @param port
 * @param path
 * @return
 */
public static FileStatus[] getFilesList(String host, int port, String path) {
    if(fs == null) {
        getFs(host, port);
    }
    Path hdfs_path = new Path(path);
    FileStatus[] path_list = null;
    try {
        path_list = fs.listStatus(hdfs_path);
        fs.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return path_list;
}

/**
 * 关闭输入字符流
 *
 * @param reader
 */
public static void reader_close(Reader reader) {
    if (reader != null) {
        try {
            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

/**
 * 关闭输出流
 *
 * @param out
 */
public static void writer_close(OutputStream out) {
    if (out != null) {
        try {
            out.close();
        } catch (IOException e) {
            logger.error("关闭输出流失败");
            e.printStackTrace();
        }
    }
}