java web整合hadoop,Spring boot 与 hadoop整合

Spring boot 与 hadoop 整合

获取目录

HadoopTest.java

import org.apache.hadoop.fs.FileStatus;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.boot.CommandLineRunner;

import org.springframework.boot.SpringApplication;

import org.springframework.boot.autoconfigure.SpringBootApplication;

import org.springframework.data.hadoop.fs.FsShell;

@SpringBootApplication

public class HadoopTest implements CommandLineRunner {

@Autowired

private FsShell shell;

[@Override](https://my.oschina.net/u/1162528)

public void run(String... args) {

for (FileStatus s : shell.lsr("/")) {

System.out.println("> " + s.getPath());

}

}

public static void main(String[] args) {

SpringApplication.run(HadoopTest.class, args);

}

}

application.yml

spring:

main:

show_banner: false

hadoop:

fsUri: hdfs://172.16.100.91:8020

pom.xml

org.springframework.data

spring-data-hadoop

2.2.0.RELEASE

org.springframework

spring-context-support

org.slf4j

slf4j-log4j12

org.apache.logging.log4j

log4j-to-slf4j

org.apache.hadoop

hadoop-client

2.7.0

org.apache.logging.log4j

log4j-to-slf4j

org.apache.hadoop

hadoop-common

${hadoop.version}

compile

org.apache.logging.log4j

log4j-to-slf4j

org.springframework.data

spring-data-hadoop-boot

2.2.0.RELEASE

org.apache.hadoop

hadoop-client

2.6.0-cdh5.7.0

Windows下需要hadoop运行环境

添加环境变量

变量名：HADOOP_USER_NAME

变量值：root

变量名：HADOOP_HOME

变量值：D:\lwenhao\apache-hadoop-3.1.0-winutils-master

需要把HADOOP_HOME配置到PATH中

提取码：suik