spring hadoop配置

为什么80%的码农都做不了架构师？>>>   

1.maven pom文件添加

<dependency>
  <groupId>org.apache.hadoop</groupId>
  <artifactId>hadoop-common</artifactId>
  <scope>compile</scope>
  <exclusions>
    <exclusion>
      <groupId>log4j</groupId>
      <artifactId>log4j</artifactId>
    </exclusion>
    <exclusion>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-log4j12</artifactId>
    </exclusion>
  </exclusions>
</dependency>
<dependency>
  <groupId>org.apache.hbase</groupId>
  <artifactId>hbase-client</artifactId>
  <scope>compile</scope>
  <exclusions>
    <exclusion>
      <groupId>log4j</groupId>
      <artifactId>log4j</artifactId>
    </exclusion>
    <exclusion>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-log4j12</artifactId>
    </exclusion>
  </exclusions>
</dependency>

<dependency>
  <groupId>org.springframework.data</groupId>
  <artifactId>spring-data-hadoop-hbase</artifactId>
</dependency>

<dependency>
  <groupId>org.springframework.data</groupId>
  <artifactId>spring-data-hadoop</artifactId>
</dependency>

 

2.添加一个配置类

@Configuration
@PropertySource("classpath:application.yml")
public class HbaseConfig {

    @Value("${hadoop.defaultFS.url}")
    String defaultFS;

    @Value("${hbase.zk.quorum}")
    String zkQuorum;            //集群列表地址

    @Value("${hbase.zk.port}")
    String zkPort;

    @Value("${hbase.zk.znode.parent}")
    String zkZnodeParent; //zookeeper 使用的根目录地址

    @Bean(value = "hadoopConfiguration")
    public org.apache.hadoop.conf.Configuration getHadoopConfig(){
        org.apache.hadoop.conf.Configuration hadConfig = new org.apache.hadoop.conf.Configuration();
        hadConfig.addResource(defaultFS);
        return hadConfig;
    }

    @Bean(value = "hbaseConfiguration")
    public org.apache.hadoop.conf.Configuration getHbaseConfig(){
        try {
            return HBaseConfiguration.createClusterConf(getHadoopConfig(),
                zkQuorum + ":" + zkPort + ":" + zkZnodeParent
            );
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }

    @Bean(value = "hbaseTemplate")
    public HbaseTemplate getHbaseTemplate(){
        return new HbaseTemplate(getHbaseConfig());
    }

}

3.yml文件配置

hadoop:
  defaultFS:
    url:  fs.defaultFS=hdfs://127.0.0.1:9000

hbase:
  zk:
    quorum: localhost
    port: 2181
    znode:
      parent: /hbase

 

转载于:https://my.oschina.net/wii01/blog/994201