Java客户端代码操作hadoop:API方式和流方式，上传、下载、文件和文件夹的创建和重命名、查看文件及文件夹信息

注意要点：代码中导包一定要正确，大部分为 org.apache.hadoop.fs.*，

Java客户端操作hadoop代码：

1、idea创建maven工程，pom文件内容为：

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.wensi.hadoop</groupId>
    <artifactId>hadoop-project</artifactId>
    <version>1.0-SNAPSHOT</version>

<dependencies>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>2.7.4</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>2.7.4</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.7.4</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.12</version>
        <scope>compile</scope>
    </dependency>
</dependencies>
</project>

2、maven刷新导入相应jar包。

3、为了解决控制台报日志异常，需要添加日志文件log4j.properties：

# Configure logging for testing: optionally with log file
#log4j.rootLogger=WARN, stdout
log4j.rootLogger=WARN, stdout, logfile

log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n

log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

log4j.logger.org.apache.hadoop.util.NativeCodeLoader=DEBUG

4、Java代码：

package com.wensi.hdfs;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Before;
import org.junit.Test;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;

public class HdfsClient {
    FileSystem fs = null;

    @Before
    public void init() throws Exception{
        Configuration conf = new Configuration();
        /**
         * 参数优先级：1、客户端代码中设置的值，2、classpath下用户自定义配置文件，3、然后是jar中默认
         */
        fs = FileSystem.get(new URI("hdfs://node-1:9000"),conf,"root");
    }

    /**
     * 往hdfs上传文件
     */
    @Test
    public void testAddFileToHdfs(){
        // 要上传的文件所在的本地路径
        try {
            Path src = new Path("D:/HadoopTest/编译Hadoop源码.txt");
            // 要上传到hdfs的目标路径
            Path dest = new Path("/");
            fs.copyFromLocalFile(src,dest);
            fs.close();
        }catch (Exception e){
            System.out.println("---------异常-------------");
        }

    }

    /**
     * 从hdfs中复制文件到本地文件系统
     */
    @Test
    public void testDownLoadFileToLocal() throws Exception{
//        fs.copyToLocalFile(new Path("/aa/bb/1.txt"),new Path("D://"));
        fs.copyToLocalFile(false,new Path("/aa/bb/1.txt"),new Path("D:/HadoopTest"));
        fs.close();
    }
    /**
     * 目录操作
     */
    @Test
    public void testMkdirAndDeleteAndRename() throws Exception{
        // 创建目录
        fs.mkdirs(new Path("/a1/b1/c1"));
        // 删除文件夹，如果是非空文件夹，参数2必须给值true
//        fs.delete(new Path("/a1"),true);
        // 重命名文件或文件夹
        fs.rename(new Path("/a1"),new Path("/a2"));
    }
    /**
     * 查看目录信息，只显示文件
     */
    @Test
    public void testListFiles() throws FileNotFoundException, IOException {
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
        while (listFiles.hasNext()){
            LocatedFileStatus fileStatus = listFiles.next();
            System.out.println(fileStatus.getBlockSize());
            System.out.println(fileStatus.getPermission());
            System.out.println(fileStatus.getLen());
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            for (BlockLocation bl:blockLocations) {
                System.out.println("block-length:"+bl.getLength()+"--"+"block-offset:"+bl.getOffset());
                String[] hosts = bl.getHosts();
                for (String host:hosts) {
                    System.out.println(host);
                }
            }
        }
    }

    /**
     * 查看文件及文件夹信息
     */
    @Test
    public void testListAll()throws FileNotFoundException,IOException{
        FileStatus[] listStatus = fs.listStatus(new Path("/"));
        String flag = "";
        for (FileStatus fsStatus:listStatus) {
            if (fsStatus.isFile()){
                flag="f--";
            }else {
                flag="d--";
            }
            System.out.println(flag+fsStatus.getPath().getName());
            System.out.println(fsStatus.getPermission());
        }
    }
    /**
     * 流形式操作,从本地复制文件到hdfs
     * 流形式效率高于API方式，但一般用API方式多余流
     */
    @Test
    public void testUpload() throws Exception{
        FSDataOutputStream outputStream = fs.create(new Path("/aa.txt"), true);
        FileInputStream inputStream = new FileInputStream("D:\\HadoopTest\\编译Hadoop源码.txt");
        IOUtils.copy(inputStream,outputStream);
    }
}