package sitech; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.mapreduce.TableMapper; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; import java.io.IOException; public class HbaseToHdfs { public static class HbaseToHdfsMapper extends TableMapper<NullWritable, Text> { private Text outPut = new Text(); @Override protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException { // StringBuffer value_res = new StringBuffer(); for (KeyValue kv : value.raw()) { String clumn_value = new String(kv.getValue(), "utf-8"); value_res.append(clumn_value + ","); } value_res.delete(value_res.length()-1,value_res.length()); outPut.set(value_res.toString()); context.write(NullWritable.get(), outPut); } } public static Job createSubmittableJob(Configuration conf, String[] args) throws IOException { String hbase_table_name = args[0]; String outPut = args[1]; conf.set("hbase_table_name", hbase_table_name); conf.set("outPut", outPut); Job job = Job.getInstance(conf, "HbaseToHdfs"); job.setJarByClass(HbaseToHdfs.class); job.setMapperClass(HbaseToHdfsMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setNumReduceTasks(0); TableMapReduceUtil.initTableMapperJob(hbase_table_name, new Scan(), HbaseToHdfsMapper.class, NullWritable.class, Text.class, job); Path path = new Path(outPut); FileSystem fs = FileSystem.get(conf); if (fs.exists(path)) { fs.delete(path, true); } FileOutputFormat.setOutputPath(job, path); return job; } public static void main(String[] args) throws InterruptedException, IOException, ClassNotFoundException { Configuration conf = HBaseConfiguration.create(); //设置压缩格式 conf.set("mapreduce.output.fileoutputformat.compress", "false"); //设置内存 conf.set("mapreduce.map.memory.mb", "2048"); String[] otherArgs = new GenericOptionsParser(conf, args) .getRemainingArgs(); if (otherArgs.length != 2) { System.exit(-1); } Job job = createSubmittableJob(conf, otherArgs); System.exit(job.waitForCompletion(true) ? 0 : 1); } }
MapReduce hbaseToHdfs
最新推荐文章于 2021-03-16 15:50:42 发布