Spring Boot 配置Hadoop
核心
- FsShell自动注入
项目工程
- Maven工程
Maven的setting.xml文件配置
。。。
<mirror>
<id>nexus</id>
<mirrorOf>*,!cloudera</mirrorOf>
<url>http://repo1.maven.org/maven2/</url>
</mirror>
。。。
pom配置
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.peng</groupId>
<artifactId>hdfstest</artifactId>
<version>1.0-SNAPSHOT</version>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
<!--配置下载的仓库-->
<repositories>
<repository>
<id>repo</id>
<url>http://repo1.maven.org/maven2/</url>
</repository>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/content/repositories/releases/</url>
</repository>
</repositories>
<!--配置依赖【注:源pom文件有冲突的依赖配置进行手动删除】-->
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.0-cdh5.7.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<version>6.1.26</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.25</version>
<scope>test</scope>
</dependency>
<!--spring boot hadoop start-->
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-hadoop-boot</artifactId>
<version>2.5.0.RELEASE</version>
</dependency>
<!--spring boot hadoop end-->
</dependencies>
</project>
配置文件【resources下】
- application.properties【配置文件,映射键值对】
spring.hadoop.fsUri=hdfs://centos00:8020
测试程序
-
SpringBootHadoopTest.java
package com.peng; import org.apache.hadoop.fs.FileStatus; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.data.hadoop.fs.FsShell; @SpringBootApplication public class SpringBootHadoopTest implements CommandLineRunner { @Autowired FsShell fsShell; @Override public void run(String... strings) throws Exception { System.out.println("=========run start============"); for (FileStatus fileStatus : fsShell.lsr("/data")) { System.out.println(">" + fileStatus.getPath()); } System.out.println("===========run end==========="); } public static void main(String[] args) { System.out.println("===========main start==========="); SpringApplication.run(SpringBootHadoopTest.class, args); System.out.println("===========main end==========="); } }
执行结果
===========main start===========
log4j:WARN No appenders could be found for logger (org.springframework.core.env.StandardEnvironment).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
. ____ _ __ _ _
/\\ / ___'_ __ _ _(_)_ __ __ _ \ \ \ \
( ( )\___ | '_ | '_| | '_ \/ _` | \ \ \ \
\\/ ___)| |_)| | | | | || (_| | ) ) ) )
' |____| .__|_| |_|_| |_\__, | / / / /
=========|_|==============|___/=/_/_/_/
:: Spring Boot :: (v1.4.7.RELEASE)
=========run start============
>hdfs://centos00:8020/data
>hdfs://centos00:8020/data/10000_access.log
>hdfs://centos00:8020/data/8_access.log
>hdfs://centos00:8020/data/hello.txt
===========run end===========
===========main end===========
Process finished with exit code 0