HBase 总结之HBase作为输入源

简介：HBase 作为输入源，即从HBase中读取数据，使用MapReduce计算完成之后，将数据存储到其他介质中。

---

下面直接上代码：

---

• 主程序

package apache.org.myhbase.asinput;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider.Text;

/**
 * hbase 作为输入数据源，即从hbase中读取内容后做处理
 * 
 * @author andrew
 *
 */
public class Main {
    static final Log LOG = LogFactory.getLog(Main.class);

    public static final String NAME = "Example Test1";
    public static final String TEMP_INDEX_PATH = "/out/example";
    public static String inputTable = "access-log";

    public static void main(String[] args) throws Exception {
        Configuration conf = HBaseConfiguration.create();

        Scan scan = new Scan();

        /*batch and caching*/
        scan.setBatch(0);
        scan.setCaching(1000);
        scan.setMaxVersions();
//      scan.setTimeRange(System.currentTimeMillis() - 3*24*3600*1000L, System.currentTimeMillis());

        /*configure scan*/
        scan.addColumn(Bytes.toBytes("info"), Bytes.toBytes("url"));

        //hbase master
        conf.set("hbase.master", "hadoop1:60000");

        //zookeeper qurom
        conf.set("hbase.zookeeper.quorum", "hadoop1,hadoop2,hadoop3");

        //set hadoop spculative execution to false
//      conf.setBoolean("mapred.map.tasks.speculative.execution", false);
//      conf.setBoolean("mapred.reudce.tasks.speculative.execution", false);

        Path tempIndexPath = new Path(TEMP_INDEX_PATH);
        FileSystem fs = FileSystem.get(conf);
        if(fs.exists(tempIndexPath)){
            fs.delete(tempIndexPath,true);
        }

        /* JOB */
        Job job = Job.getInstance(conf,NAME);
//      Job job = new Job(conf, NAME);
        job.setJarByClass(Main.class);

        TableMapReduceUtil.initTableMapperJob(inputTable, scan, ExampleMapper.class, 
                Text.class, Text.class, job);

        job.setNumReduceTasks(0);
        job.setOutputFormatClass(TextOutputFormat.class);
        FileOutputFormat.setOutputPath(job, tempIndexPath);

        int success = job.waitForCompletion(true) ? 0:1;

        System.exit(success);
    }
}

---

• 这里是主程序中配置的Mapper

package apache.org.myhbase.asinput;

import java.io.IOException;

import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;

/**
 * From HBase Book，读取hbase 中的数据后处理的Mapper
 * 
 * @author 13277
 *
 */
public class ExampleMapper extends TableMapper<Writable, Writable> {

    final static String inTable = "access-log";
    final static String inFamily = "info";
    final static String inColumn = "url";

    final static  String outTable="total-access";
    final static  String outFamily="url";
    final static  String outColumn="count";

    public static final String FIELD_COMMON_SEPARATOR = "\u0001";

    @Override
    protected void map(ImmutableBytesWritable row, Result values, Context context)
            throws IOException, InterruptedException {

        byte[] b = values.getValue(inFamily.getBytes(), inColumn.getBytes());

        if (b != null) {
            String v = new String(b);
            String r = new String(values.getRow());
            String[] strs = r.split("-");
            String ip = strs[0];
            System.out.println("iamhere" + r);
            context.write(new Text(ip), new Text(v));
        }
    }

    @Override
    protected void setup(Context context) throws IOException, InterruptedException {

        super.setup(context);
    }
}

---

PS：
在主程序中，Hadoop任务配置中的 TableMapReduceUtil.initTableMapperJob()方法设置了数据输入表（access-log）,扫描实例（scan），方法内部设置输入格式类型为TextInputFormat。流程中并没有设置Reducer，计算结果的数据使用TextOutputFormat类，即将最终结果存放到HDFS上。