使用mr将hbase数据存入mysql中

这个是mr程序

package com.ygy.mr.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.*;

import java.io.IOException;

public class mytest {

    //map阶段
    public static class MyTestMap extends TableMapper<Text,Text>{
        @Override
        protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {

            context.write(new Text(key.get()),new Text(value.value()));
        }
    }

    //reduce阶段
    public static class MyTestReducer extends Reducer<Text,Text,Text,Text> {
        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {

            for (Text value : values) {
                context.write(key,value);
            }
        }
    }


    public static void main(String[] args) throws Exception {
        //表名
        String tableName = "mytest";
        //获取resource配置
        Configuration conf = HBaseConfiguration.create();
        //job
        Job job = Job.getInstance(conf);
        job.setJarByClass(mytest.class);
        //hbase链接
        Connection connection = ConnectionFactory.createConnection(job.getConfiguration());
        Admin admin = connection.getAdmin();
        //判断一下
        if (admin.tableExists(TableName.valueOf(tableName))) {
            System.out.println("存在");
        }
        //查看数据
        Scan scan = new Scan();
        //组件
        TableMapReduceUtil.initTableMapperJob(
                tableName,
                scan,
                MyTestMap.class,
                Text.class,
                Text.class,
                job,
                true
        );
        //reducer
        job.setReducerClass(MyTestReducer.class);
        //设置处处的key
        job.setOutputKeyClass(Text.class);
        //设置输出的value
        job.setOutputValueClass(Text.class);
        //输出
        job.setOutputFormatClass(MyOutputFormatClass.class);
        //本地
        //FileOutputFormat.setOutputPath(job,new Path("out1"));
        boolean b = job.waitForCompletion(true);
        System.out.println(b);

    }
}

这个是自定义输出

package com.ygy.mr.test;


import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;

public  class MyOutputFormatClass extends OutputFormat<Text,Text> {
    private OutputCommitter committer = null;

    public static class MySqlWrite extends RecordWriter<Text,Text> {
        private Connection con;
        public MySqlWrite(Connection connection) {

            this.con=connection;
        }

        @Override
        public void write(Text key, Text value) throws IOException, InterruptedException {
            try {
                String innerSql = "insert into t1 values(?,?) ";
                PreparedStatement preparedStatement = con.prepareStatement(innerSql);
                preparedStatement.setString(1,key.toString());
                preparedStatement.setString(2,value.toString());
                preparedStatement.execute();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }

        @Override
        public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
            try {
                con.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }

    @Override
    public RecordWriter<Text, Text> getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
        //初始化jdbc
        Connection  connection=null;
            try {
                Class.forName("com.mysql.jdbc.Driver");
                connection = DriverManager.getConnection("jdbc:mysql://hadoop102:3306/test?useUnicode=true&characterEncoding=UTF-8", "root", "000000");
            } catch (ClassNotFoundException e) {
                e.printStackTrace();
            } catch (SQLException e) {
                e.printStackTrace();
            }

        return new MySqlWrite(connection);
    }

    @Override
    public void checkOutputSpecs(JobContext jobContext) throws IOException, InterruptedException {
        System.out.println("go");
    }

    @Override
    public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {

        return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),context);
    }

}

建议mr两个阶段分开写，增加耦合性