hbase通过MapperReducer进行批量插入数据

MapperReducer插入数据

package com.hbase.test02;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class MRInsertData {

public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
	System.setProperty("HADOOP_USER_NAME", "hadoop11");

	Configuration conf = new Configuration();
	// 设置zookeeper通信
	conf.set("hbase.zookeeper.quorum", "hadoop01:2181,hadoop02:2181,hadoop03:2181");
	conf.set("fs.defaultFS", "hdfs://bd1901/");
	// 启动job
	Job job = Job.getInstance(conf);

	// 设置jar主类
	job.setJarByClass(MRInsertData.class);
	// 设置map类
	job.setMapperClass(MyMapper.class);

	// 借助工具类设置输出的路径
	TableMapReduceUtil.initTableReducerJob("habseBD:test04", MyReducer.class, job, null, null, null, null, false);
	job.setMapOutputKeyClass(Text.class);
	job.setMapOutputValueClass(NullWritable.class);
	job.setOutputKeyClass(NullWritable.class);
	job.setOutputValueClass(Put.class);
	// hdfs输入路径
	FileInputFormat.addInputPath(job, new Path("/ll/ratings"));
	// 提交
	job.waitForCompletion(true);

}

static class MyMapper extends Mapper<LongWritable, Text, Text, NullWritable> {
	Text mk = new Text();

	@Override
	protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, NullWritable>.Context context)
			throws IOException, InterruptedException {

//获取一行数据
String[] str = value.toString().split(":?;
if (str.length == 4) {
mk.set(str[0] + “\t” + str[1] + “\t” + str[2] + “\t” + str[3]);
context.write(mk, NullWritable.get());
}

	}
}

//继承TableReducer类只有三个参数是因为输出的valus是默认的不用指定
static class MyReducer extends TableReducer<Text, NullWritable, NullWritable> {
@Override
protected void reduce(Text key, Iterable values, Context context)
throws IOException, InterruptedException {
String[] str = key.toString().split("\t");
// 封装put队像 行健具有唯一性
context.getCounter(MyCounter.ROW_KEY).increment(1L);
// 获取行健值
long value = context.getCounter(MyCounter.ROW_KEY).getValue();
Put p = new Put((“rk” + value).getBytes());
// “zu01”.getBytes()列族
p.addColumn(“zu01”.getBytes(), “userid”.getBytes(), str[0].getBytes());
p.addColumn(“zu01”.getBytes(), “movieid”.getBytes(), str[1].getBytes());
p.addColumn(“zu01”.getBytes(), “rate”.getBytes(), str[2].getBytes());
p.addColumn(“zu01”.getBytes(), “ts”.getBytes(), str[3].getBytes());
// 发送输出结果到hbase中
context.write(NullWritable.get(), p);

	}

}

}

//设置枚举类定义行健
enum MyCounter {
ROW_KEY
}