简介:HBase 作为输入源,即从HBase中读取数据,使用MapReduce计算完成之后,将数据存储到其他介质中。
下面直接上代码:
- 主程序
package apache.org.myhbase.asinput;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider.Text;
/**
* hbase 作为输入数据源,即从hbase中读取内容后做处理
*
* @author andrew
*
*/
public class Main {
static final Log LOG = LogFactory.getLog(Main.class);
public static final String NAME = "Example Test1";
public static final String TEMP_INDEX_PATH = "/out/example";
public static String inputTable = "access-log";
public static void main(String[] args) throws Exception {
Configuration conf = HBaseConfiguration.create();
Scan scan = new Scan();
/*batch and caching*/
scan.setBatch(0);
scan.setCaching(1000);
scan.setMaxVersions();
// scan.setTimeRange(System.currentTimeMillis() - 3*24*3600*1000L, System.currentTimeMillis());
/*configure scan*/
scan.addColumn(Bytes.toBytes("info"), Bytes.toBytes("url"));
//hbase master
conf.set("hbase.master", "hadoop1:60000");
//zookeeper qurom
conf.set("hbase.zookeeper.quorum", "hadoop1,hadoop2,hadoop3");
//set hadoop spculative execution to false
// conf.setBoolean("mapred.map.tasks.speculative.execution", false);
// conf.setBoolean("mapred.reudce.tasks.speculative.execution", false);
Path tempIndexPath = new Path(TEMP_INDEX_PATH);
FileSystem fs = FileSystem.get(conf);
if(fs.exists(tempIndexPath)){
fs.delete(tempIndexPath,true);
}
/* JOB */
Job job = Job.getInstance(conf,NAME);
// Job job = new Job(conf, NAME);
job.setJarByClass(Main.class);
TableMapReduceUtil.initTableMapperJob(inputTable, scan, ExampleMapper.class,
Text.class, Text.class, job);
job.setNumReduceTasks(0);
job.setOutputFormatClass(TextOutputFormat.class);
FileOutputFormat.setOutputPath(job, tempIndexPath);
int success = job.waitForCompletion(true) ? 0:1;
System.exit(success);
}
}
- 这里是主程序中配置的Mapper
package apache.org.myhbase.asinput;
import java.io.IOException;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
/**
* From HBase Book,读取hbase 中的数据后处理的Mapper
*
* @author 13277
*
*/
public class ExampleMapper extends TableMapper<Writable, Writable> {
final static String inTable = "access-log";
final static String inFamily = "info";
final static String inColumn = "url";
final static String outTable="total-access";
final static String outFamily="url";
final static String outColumn="count";
public static final String FIELD_COMMON_SEPARATOR = "\u0001";
@Override
protected void map(ImmutableBytesWritable row, Result values, Context context)
throws IOException, InterruptedException {
byte[] b = values.getValue(inFamily.getBytes(), inColumn.getBytes());
if (b != null) {
String v = new String(b);
String r = new String(values.getRow());
String[] strs = r.split("-");
String ip = strs[0];
System.out.println("iamhere" + r);
context.write(new Text(ip), new Text(v));
}
}
@Override
protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
}
}
PS:
在主程序中,Hadoop任务配置中的 TableMapReduceUtil.initTableMapperJob()方法设置了数据输入表(access-log),扫描实例(scan),方法内部设置输入格式类型为TextInputFormat。流程中并没有设置Reducer,计算结果的数据使用TextOutputFormat类,即将最终结果存放到HDFS上。