Hadoop的javaAPI客户端HDFS配置使用详解

环境配置

1. 将hadoop解压到window中,路径不能有中文,不能有空格。
2. 拷贝hadoop的windows执行环境工具到bin路径下,替换客户端工具。
3. 配置环境变量
	HADOOP_HOME=hadoop安装路径
	PATH=hadoop安装路径/bin
4. 重启IDEA或者eclipse

Hadoop项目maven依赖

<dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>3.3.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.3.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs-client</artifactId>
            <version>3.3.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.3.1</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13.2</version>
            <scope>test</scope>
        </dependency>
        <!-- log4j依赖 -->
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.17.2</version>
        </dependency>
    </dependencies>

增加日志配置文件

Log4j 支持两种配置文件格式，一种是 XML 格式的文件，一种是 Java 特性文件 lg4j.properties （键 = 值）

log4j中的xml配置文件要优先于properties文件

（1）log4j2.xml

在 src/main/resources 目录中增加日志配置文件 log4j2.xml

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="error" strict="true" name="XMLConfig">
    <Appenders>
        <Appender type="Console" name="STDOUT">
            <Layout type="PatternLayout"
                    pattern="[%p] [%d{yyyy-MM-dd HH:mm:ss}][%c{10}]%m%n" />
        </Appender>
    </Appenders>

    <Loggers>
        <!-- 可加性为false -->
        <Logger name="test" level="info" additivity="false">
            <AppenderRef ref="STDOUT" />
        </Logger>

        <!-- root loggerConfig设置 -->
        <Root level="info">
            <AppenderRef ref="STDOUT" />
        </Root>
    </Loggers>
</Configuration>

（2）lg4j.properties

### 设置###
log4j.rootLogger = debug,stdout,D,E

### 输出信息到控制抬 ###
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n

### 输出DEBUG 级别以上的日志到 ###
log4j.appender.D = org.apache.log4j.DailyRollingFileAppender
log4j.appender.D.File = D://linuxvmware/hadooptest2/src/main/resources/data/log.log
log4j.appender.D.Append = true
log4j.appender.D.Threshold = DEBUG 
log4j.appender.D.layout = org.apache.log4j.PatternLayout
log4j.appender.D.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss}  [ %t:%r ] - [ %p ]  %m%n

### 输出ERROR 级别以上的日志到=error.log ###
log4j.appender.E = org.apache.log4j.DailyRollingFileAppender
log4j.appender.E.File =D://linuxvmware/hadooptest2/src/main/resources/data/error.log 
log4j.appender.E.Append = true
log4j.appender.E.Threshold = ERROR 
log4j.appender.E.layout = org.apache.log4j.PatternLayout
log4j.appender.E.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss}  [ %t:%r ] - [ %p ]  %m%n

执行代码

        System.setProperty("HADOOP_USER_NAME","root");

        Configuration conf = new Configuration();


        conf.set("fs.defaultFS","hdfs://192.168.244.100:8020");

        conf.set("dfs.client.use.datanode.hostname", "true");
        FileSystem fs = FileSystem.get(conf);


        //创建目录
        Boolean flag = fs.mkdirs(new Path("/root/data"));
        System.out.println(flag);

        //判断文件是否存在
        Boolean isExists = fs.exists(new Path("/root/data/student2.txt"));
        System.out.println(isExists);
        if(isExists == false){
            fs.create(new Path("/root/data/student2.txt"));
        }



        //查看文件信息
        FileStatus fileStatus = fs.getFileStatus(new Path("/root/data/student2.txt"));
        System.out.println(fileStatus.toString());

        //下载文件
        if(isExists == false){
            fs.copyToLocalFile(new Path("/root/data/"),new Path("D:\\linuxvmware\\hadooptest2\\src\\main\\resources\\"));
        }


        //上传文件

        String srcFilePath = "C:\\Users\\86132\\Desktop\\0715.txt";
        String remoteFilePath = "/root/data/";
        fs.copyFromLocalFile(new Path(srcFilePath),new Path(remoteFilePath));

        //删除文件
        //fs.delete(new Path("/root/data/"),true);


        //查看文件列表
        RemoteIterator<LocatedFileStatus> fileList = fs.listFiles(new Path("/root/data/"), true);
        while (fileList.hasNext()){
            System.out.println(fileList.next().toString());
        }

Google Option概要及使用

安装

<dependency>
  <groupId>com.github.pcj</groupId>
  <artifactId>google-options</artifactId>
  <version>1.0.0</version>
</dependency>

创建扩展和定义的解析器类

 
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionsBase;
 
import java.util.List;
 
/**
 * Command-line options definition for example server.
 */
public class ServerOptions extends OptionsBase {
 
    @Option(
            name = "help",
            abbrev = 'h',
            help = "Prints usage info.",
            defaultValue = "true"
    )
    public boolean help;
 
    @Option(
            name = "host",
            abbrev = 'o',
            help = "The server host.",
            category = "startup",
            defaultValue = ""
    )
    public String host;
 
    @Option(
            name = "port",
            abbrev = 'p',
            help = "The server port.",
            category = "startup",
            defaultValue = "8080"
    )
    public int port;
 
    @Option(
            name = "dir",
            abbrev = 'd',
            help = "Name of directory to serve static files.",
            category = "startup",
            allowMultiple = true,
            defaultValue = ""
    )
    public List<String> dirs;
 
}

解析器类的参数使用

 
import com.google.devtools.common.options.OptionsParser;
import java.util.Collections;
 
public class Server {
 
    //java -jar server.jar --host=127.0.0.1 --port=21 --dir=E:\test --dir=E:\demo
    public static void main(String[] args) {
        OptionsParser parser = OptionsParser.newOptionsParser(ServerOptions.class);
        parser.parseAndExitUponError(args);
        ServerOptions options = parser.getOptions(ServerOptions.class);
        if (options.host.isEmpty() || options.port < 0 || options.dirs.isEmpty()) {
            printUsage(parser);
            return;
        }
 
        System.out.format("Starting server at %s:%d...\n", options.host, options.port);
        for (String dirname : options.dirs) {
            System.out.format("\\--> Serving static files at <%s>\n", dirname);
        }
    }
 
    private static void printUsage(OptionsParser parser) {
        System.out.println("Usage: java -jar server.jar OPTIONS");
        System.out.println(parser.describeOptions(Collections.<String, String>emptyMap(),
                OptionsParser.HelpVerbosity.LONG));
    }
 
}