【代码样板】HBase一个MapReduce程序ETL

public class SaleOrdersMapReducer extends Configured implements Tool {
	//原表：数据来源
    private final static String ORDERS_TABLE_NAME="ns1:orders";
    //处理后的数据，表需要提前创建
    private final static String HISTORY_ORDERS_TABLE_NAME="orders:history_orders";
    static class ReadOrderMapper extends TableMapper<ImmutableBytesWritable,Put>{
    	//定义基本的不变的字段
        private final static String ORDER_COLUMN_NAME_USER_ID = "user_id";
        private final static String ORDER_COLUMN_NAME_ORDER_ID = "order_id";
        private final static String ORDER_COLUMN_NAME_DATE = "date";
        private final static String HISTORY_ROW_KEY_SEPARATOR = "_";
        private final static byte[] HISTORY_COLUMN_FAMILY= Bytes.toBytes( "order" );

        private ImmutableBytesWritable mapOutput = new ImmutableBytesWritable(  );
        @Override
        protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {

            //编写专门的方法，转换数据，得到Put对象
            Put put = resultToPut(key,value);
            //输出rowKey
            mapOutput.set( put.getRow() );
            //输出
            context.write( mapOutput,put );
        }

        private Put resultToPut(ImmutableBytesWritable key, Result result) {
            //订单Id
            String orderId = Bytes.toString( key.get() );
            //date,user_id,order_amt
            HashMap<String, String> orderMap = new HashMap<>();
            for (Cell cell:result.rawCells()) {
                String filed = Bytes.toString(CellUtil.cloneQualifier( cell ));
                String value = Bytes.toString(CellUtil.cloneValue( cell ));
                orderMap.put( filed ,value);
            }
            //组合rowKey:userId + orderDate + orderId
            StringBuffer sb = new StringBuffer();
            //reverse(userId)
            sb.append( orderMap.get( ORDER_COLUMN_NAME_USER_ID ) ).reverse();
            sb.append( HISTORY_ROW_KEY_SEPARATOR );
            //date
            sb.append( orderMap.get( ORDER_COLUMN_NAME_DATE )  );
            sb.append( HISTORY_ROW_KEY_SEPARATOR );
            sb.append( orderId );
            //创建Put对象
            Put put = new Put(Bytes.toBytes( sb.toString() ));
            for (Map.Entry<String,String> entry:orderMap.entrySet()) {
                put.addColumn(
                        HISTORY_COLUMN_FAMILY,
                        Bytes.toBytes( entry.getKey() ),
                        Bytes.toBytes( entry.getValue() )   );
            }
            put.addColumn(
                    HISTORY_COLUMN_FAMILY,
                    Bytes.toBytes( ORDER_COLUMN_NAME_ORDER_ID ),
                    Bytes.toBytes( orderId ));
            return put;
        }
    }
    @Override
    public int run(String[] args) throws Exception {
        //读取配置
        Configuration conf = this.getConf();
        //创建Job
        Job job = Job.getInstance( conf, SaleOrdersMapReducer.class.getName() );
        job.setJarByClass( SaleOrdersMapReducer.class );
        //设置Job：
        //input:table  ->map ->output:table
        Scan scan = new Scan();
        // 1 is the default in Scan, which will be bad for MapReduce jobs
        scan.setCaching(500);
        // don't set to true for MR jobs
        scan.setCacheBlocks(false);
        //设置Mapper类和Input table
        TableMapReduceUtil.initTableMapperJob(
                ORDERS_TABLE_NAME,        // input HBase table name
                scan,             // Scan instance to control CF and attribute selection
                ReadOrderMapper.class,   // mapper
                ImmutableBytesWritable.class, // mapper output key，RowKey
                Put.class,        // mapper output value,行内容
                job);
        //设置输出以及Reducer
        TableMapReduceUtil.initTableReducerJob(
                HISTORY_ORDERS_TABLE_NAME,      // output table
                null,             // reducer class
                job);
        job.setNumReduceTasks(0);

        //如果数据量非常大的情况下，不建议使用put方式将数据插入到HBASE表中，
        //而是将数据转成HBASE数据存储的HFile

        //设置MapReduce输出的数据格式
        job.setOutputFormatClass( HFileOutputFormat2.class );

        //往那张表里面写
        HTable table = new HTable( conf, HISTORY_ORDERS_TABLE_NAME );
        HFileOutputFormat2.configureIncrementalLoad( job, table,table.getRegionLocator());

        //设置HFile文件的输出目录
        Path outputPath = new Path(args[0] + System.currentTimeMillis());
        FileOutputFormat.setOutputPath( job, outputPath);

        boolean isSuccess = job.waitForCompletion( true );
        //如果MapReduce运行完成，成功之后，将输出HFile文件 加载到 表中
        if(isSuccess){
            LoadIncrementalHFiles load = new LoadIncrementalHFiles( conf );
            load.doBulkLoad( outputPath, table);
        }
        return isSuccess?0:1;
    }


    public static void main(String[] args) {
        //HBase配置文件
        Configuration conf = HBaseConfiguration.create();
        try {
            //运行job
            int status = ToolRunner.run( conf, new SaleOrdersMapReducer(), args );
            //结束程序
            System.exit( status );
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}