packagebulkloadExample;importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.FileSystem;importorg.apache.hadoop.fs.Path;importorg.apache.hadoop.hbase.HBaseConfiguration;importorg.apache.hadoop.hbase.TableName;importorg.apache.hadoop.hbase.client.Connection;importorg.apache.hadoop.hbase.client.ConnectionFactory;importorg.apache.hadoop.hbase.client.Put;importorg.apache.hadoop.hbase.io.ImmutableBytesWritable;importorg.apache.hadoop.hbase.mapreduce.HFileOutputFormat2;importorg.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;importorg.apache.hadoop.hbase.mapreduce.PutSortReducer;importorg.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;importorg.apache.hadoop.hbase.util.Bytes;importorg.apache.hadoop.io.LongWritable;importorg.apache.hadoop.io.Text;importorg.apache.hadoop.mapreduce.Job;importorg.apache.hadoop.mapreduce.Mapper;importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;importjava.io.IOException;importjava.net.URI;public classBulkloadData {static final String JOBNAME = "BulkLoad";static final String TABLENAME = "wqbin:duowan_user_bulkLoad";static final String PATH_IN = "/datain/duowan_user.txt"; //输入路径
static final String PATH_OUT = "/dataout"; //输入路径HFILE
static final String SEPARATOR = "\t";static final byte[] ColumnFamily = "f".getBytes(); //列簇//static final byte[] row_id = "id".getBytes();//列名
static final byte[] name = "name".getBytes();static final byte[] pass = "pass".getBytes();static final byte[] mail = "mail".getBytes();static final byte[] nickname = "nickname".getBytes();public static class Map extends Mapper{protected void map(LongWritable key, Text value, Context context) throwsIOException, InterruptedException {
String[] strArr=value.toString().split(SEPARATOR);
String row= strArr[0];
Put put= new Put(Bytes.toBytes(row.toString())); //rowkey
put.add(ColumnFamily, name, Bytes.toBytes(strArr[1]));
put.add(ColumnFamily, pass, Bytes.toBytes(strArr[2]));
put.add(ColumnFamily, mail, Bytes.toBytes(strArr[3]));
put.add(ColumnFamily, nickname, Bytes.toBytes(strArr[3]));
context.write(newImmutableBytesWritable(value.getBytes()), put);
}
}public static void main(String[] args) throwsException {
Configuration conf=HBaseConfiguration.create();
Connection conn=ConnectionFactory.createConnection(conf);//conf.set("hbase.zookeeper.quorum", "xx,xx,xx");
conf.set("fs.defaultFS","file:///");
System.setProperty("HADOOP_USER_NAME", "centos");
Job job=Job.getInstance(conf, JOBNAME);
job.setJarByClass(BulkloadData.class);
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(Put.class);
TableMapReduceUtil.addDependencyJars(job);
job.setMapperClass(Map.class);//这个 SorterReducer(KeyValueSortReducer或PutSortReducer) 可以不指定,//因为源码中已经做了判断
job.setReducerClass(PutSortReducer.class);
job.setOutputFormatClass(HFileOutputFormat2.class);
FileSystem fs= FileSystem.get(URI.create("/"),conf);
Path outPath= newPath(PATH_OUT);//if (fs.exists(outPath))fs.delete(outPath, true);
FileOutputFormat.setOutputPath(job, outPath);
FileInputFormat.setInputPaths(job,newPath(PATH_IN));//HTable table = new HTable(conf, TABLENAME);
HFileOutputFormat2.configureIncrementalLoad(job, conn.getTable(TableName.valueOf(TABLENAME)),
conn.getRegionLocator(TableName.valueOf(TABLENAME)));
System.out.println("load完毕");if(job.waitForCompletion(true)){
LoadIncrementalHFiles loader= newLoadIncrementalHFiles(conf);//loader.doBulkLoad( table);
}
System.exit(0);
}
}