package com.bigdata.hbase.hdfstohbase;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
/**
* 构建ReadCityFromHDFSMapper于读取HDFS中的文件数据
*/
public class ReadCityFromHDFSMapper extends Mapper<LongWritable, Text, ImmutableBytesWritable, Put> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//从hdfs中读数据
String lineValue=value.toString();
//\t切分数据
String[] splits=lineValue.split("\t");
//读取数据中的数据
String rowkey=splits[0];
String name=splits[1];
String country=splits[2];
//初始化rowKey
ImmutableBytesWritable rowkeywriteable=new ImmutableBytesWritable(Bytes.toBytes(rowkey));
//初始化put对象
Put put=new Put(Bytes.toBytes(rowkey));
//参数分别:列族、列、值
put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"),Bytes.toBytes(name));
put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("country"),Bytes.toBytes(country));
context.write(rowkeywriteable,put);
}
}
package com.bigdata.hbase.hdfstohbase;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.NullWritable;
import java.io.IOException;
/**
*运行reduce类
*/
public class WriteCityMRFromTxtReduce extends TableReducer<ImmutableBytesWritable, Put, NullWritable> {
@Override
protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {
//读出来的每一行数据写入到city表中
for(Put put: values){
context.write(NullWritable.get(), put);
}
}
}
package com.bigdata.hbase.hdfstohbase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* 运行读hdfs数据到hbase
*/
public class HDFSToHbaseRunner extends Configured implements Tool {
@Override
public int run(String[] strings) throws Exception {
//得到Configuration
Configuration conf = this.getConf();
//创建Job任务
Job job = Job.getInstance(conf, this.getClass().getSimpleName());
job.setJarByClass(HDFSToHbaseRunner.class);
Path inPath = new Path("hdfs://bigdata112:9000//hbase_mr_test/city.tsv");
FileInputFormat.addInputPath(job, inPath);
//设置Mapper
job.setMapperClass(ReadCityFromHDFSMapper.class);
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(Put.class);
//设置Reducer
TableMapReduceUtil.initTableReducerJob("city", WriteCityMRFromTxtReduce.class, job);
//设置Reduce数量,最少1个
job.setNumReduceTasks(1);
boolean isSuccess = job.waitForCompletion(true);
return isSuccess ? 1 : 0;
}
public static void main(String[] args) throws Exception {
Configuration conf = HBaseConfiguration.create();
int status = ToolRunner.run(conf, new HDFSToHbaseRunner(), args);
System.exit(status);
}
}
运行前效果。空city表
运行后效果。