《中级》Flink 读写hbase

ReadFromHBase

import lombok.extern.slf4j.Slf4j;
import org.apache.flink.addons.hbase.TableInputFormat;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.tuple.Tuple4;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;


@Slf4j
public class TestReadFromHBase {
    public static final byte[] family = "F".getBytes(ConfigConstants.DEFAULT_CHARSET);

    public static void main(String[] args) throws Exception {
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        DataSet<Tuple4<String, String, Integer, String>> users = env.createInput(new TableInputFormat<Tuple4<String, String, Integer, String>>() {

            @Override
            public void configure(org.apache.flink.configuration.Configuration parameters) {
                table = createTable();
                if (table != null) {
                    scan = getScanner();
                }
            }

            private HTable createTable() {
                log.info("Initializing HBaseConfiguration");
                //use files found in the classpath
                org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();

                conf.set("hbase.zookeeper.quorum", "slave01,slave02,slave03");
                conf.set("hbase.zookeeper.property.clientPort", "2181");
                conf.set("zookeeper.znode.parent", "/hbase-unsecure");

                try {
                    return new HTable(conf, getTableName());
                } catch (Exception e) {
                    log.error("Error instantiating a new HTable instance", e);
                }
                return null;
            }

            @Override
            protected Scan getScanner() {
                Scan scan = new Scan();
                scan.addFamily(family);
                return scan;
            }

            @Override
            protected String getTableName() {
                return "learing_flink:users";
            }

            @Override
            protected Tuple4<String, String, Integer, String> mapResultToTuple(Result r) {
                return Tuple4.of(
                        Bytes.toString(r.getRow()),
                        Bytes.toString(r.getValue(family, "name".getBytes(ConfigConstants.DEFAULT_CHARSET))),
                        Integer.parseInt(
                                Bytes.toString(
                                        r.getValue(
                                                family, "age".getBytes(
                                                        ConfigConstants.DEFAULT_CHARSET
                                                )
                                        )
                                )
                        ),
                        Bytes.toString(r.getValue(family, "address".getBytes(ConfigConstants.DEFAULT_CHARSET)))
                );
            }
        });

        users.print();

    }
}

 

 

import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.hadoop.mapreduce.HadoopOutputFormat;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple4;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;


public class TestWriteToHBase {
    public static void main(String[] args) throws Exception{
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        DataSet<Tuple4<String, String, Integer, String>> users=env.fromElements(
                Tuple4.of("1000","zhansgan",30,"beijing"),
                Tuple4.of("1001","lisi",23,"guangdong"),
                Tuple4.of("1002","wangwu",45,"hubei"),
                Tuple4.of("1003","zhanliu",23,"beijing"),
                Tuple4.of("1004","lilei",56,"henan"),
                Tuple4.of("1005","maxiaoshuai",34,"xizang"),
                Tuple4.of("1006","liudehua",26,"fujian"),
                Tuple4.of("1007","jiangxiaohan",18,"hubei"),
                Tuple4.of("1008","qianjin",29,"shanxi"),
                Tuple4.of("1009","zhujie",37,"shandong"),
                Tuple4.of("1010","taobinzhe",19,"guangxi"),
                Tuple4.of("1011","wuqixian",20,"hainan")
                );

//        users.print();

        //生成HBase输出数据
        DataSet<Tuple2<Text, Mutation>> result = convertResultToMutation(users);

        //4.输出到HBase(直接使用hadoop的OutputFormat)

        org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.quorum", "slave01,slave02,slave03");
        conf.set("hbase.zookeeper.property.clientPort", "2181");
        conf.set("zookeeper.znode.parent", "/hbase-unsecure");
        conf.set(TableOutputFormat.OUTPUT_TABLE, "learing_flink:users");
        conf.set("mapreduce.output.fileoutputformat.outputdir", "/tmp");

        Job job = Job.getInstance(conf);

        result.output(new HadoopOutputFormat<Text, Mutation>(new TableOutputFormat<Text>(), job));

        env.execute("TestWriteToHBase");
    }

    public static DataSet<Tuple2<Text, Mutation>> convertResultToMutation(DataSet<Tuple4<String, String, Integer, String>> users) {
        return users.map(new RichMapFunction<Tuple4<String, String, Integer, String>, Tuple2<Text, Mutation>>() {

            private transient Tuple2<Text, Mutation> resultTp;

            private byte[] cf ="F".getBytes(ConfigConstants.DEFAULT_CHARSET);

            @Override public void open(Configuration parameters) throws Exception {
                super.open(parameters);
                resultTp = new Tuple2<>();
            }

            @Override
            public Tuple2<Text, Mutation> map(Tuple4<String, String, Integer, String> user) throws Exception {
                resultTp.f0 = new Text(user.f0);
                Put put = new Put(user.f0.getBytes(ConfigConstants.DEFAULT_CHARSET));

                if (null != user.f1) {
                    put.addColumn(cf, Bytes.toBytes("name"), Bytes.toBytes(user.f1));
                }

                //记得一定要先toString
                put.addColumn(cf, Bytes.toBytes("age"), Bytes.toBytes(user.f2.toString()));

                if (null != user.f3) {
                    put.addColumn(cf, Bytes.toBytes("address"), Bytes.toBytes(user.f3));
                }

                resultTp.f1 = put;
                return resultTp;
            }
        });
    }
}

加微信拉微信交流群

微信：weixin605405145