spark-写入到hbase

最新推荐文章于 2023-05-04 23:00:00 发布

cclovezbf

最新推荐文章于 2023-05-04 23:00:00 发布

阅读量377

点赞数

分类专栏： spark 文章标签： spark 写hbase

本文链接：https://blog.csdn.net/cclovezbf/article/details/116157626

版权

spark 专栏收录该内容

17 篇文章 1 订阅

订阅专栏

pom

 <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>${spark.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>org.apache.hadoop</groupId>
                    <artifactId>hadoop-client</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.apache.hadoop</groupId>
                    <artifactId>hadoop-hdfs</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
            <exclusions>
                <exclusion>
                    <artifactId>org.cloudera.logredactor</artifactId>
                    <groupId>logredactor</groupId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>${hbase.version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase-it -->
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-it</artifactId>
            <version>2.1.0-cdh6.3.3</version>
            <scope>compile</scope>
        </dependency>
        <!--spark 支持hive -->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.11</artifactId>
            <version>${spark.version}</version>
            <exclusions>
                <exclusion>
                    <artifactId>jackson-databind</artifactId>
                    <groupId>com.fasterxml.jackson.core</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>org.cloudera.logredactor</artifactId>
                    <groupId>logredactor</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>org.cloudera.logredactor</artifactId>
                    <groupId>logredactor</groupId>
                </exclusion>
            </exclusions>
            <!--<scope>provided</scope>-->
        </dependency>

java版本spark。

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapred.TableOutputFormat;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapred.JobConf;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;
import scala.Tuple3;

import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;

public class HiveToHbase {
    public static void main(String[] args) throws IOException {
        JavaSparkContext jsc = new JavaSparkContext("local[*]","11",new SparkConf());
        jsc.hadoopConfiguration().set("hbase.zookeeper.quorum ","xxx:2181,xxx:2181,xxx:2181");
        jsc.hadoopConfiguration().set("zookeeper.znode.parent","/hbase");
        jsc.hadoopConfiguration().set(TableOutputFormat.OUTPUT_TABLE,"jl:test_load");
        jsc.hadoopConfiguration().set( "mapreduce.output.fileoutputformat.outputdir", "/tmp/cc/"+ UUID.randomUUID());
        JobConf jobConf = new JobConf(jsc.hadoopConfiguration());
        jobConf.setOutputKeyClass(ImmutableBytesWritable.class);
        jobConf.setOutputValueClass(Put.class);
        jobConf.setOutputFormat(TableOutputFormat.class);
        List<Integer> data = Arrays.asList(1, 2, 3, 4, 5);
        JavaRDD<Integer> javaRDD = jsc.parallelize(data);
        javaRDD.map(n->new Tuple3<String,String,String>("key" + n, "code" + n, "level" + n))
                .mapToPair(t->{
                    ImmutableBytesWritable rowKey = new ImmutableBytesWritable(Bytes.toBytes(t._1()));
                    Put put = new Put(Bytes.toBytes(t._1()));
                    put.addColumn(Bytes.toBytes("0"), Bytes.toBytes("ou_code"), Bytes.toBytes(t._2()));
                    put.addColumn(Bytes.toBytes("0"), Bytes.toBytes("ou_level"), Bytes.toBytes(t._3()));
                    return new Tuple2(rowKey, put);
                }).saveAsHadoopDataset(jobConf);

        jsc.stop();
    }
}

最后结果