package Hbase;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
/**
* HBASE结合MapReduce批量导入
* @author 刘超
*/
public class FPL {
public static void createTable() throws MasterNotRunningException, ZooKeeperConnectionException, IOException{
//配置 必须书写
Configuration conf = HBaseConfiguration.create();
String tableName = "FPL"; //表名
String family_name = "conf"; //列族
conf.set("hbase.rootdir", "hdfs://localhost:9000/hbase");
conf.set("hbase.zookeeper.property.dataDir","/home/liuchao/Hbase/zookeeper_data");
conf.set("hbase.zookeeper.quorum","localhost");
final HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
if(!hbaseAdmin.tableExists(tableName)){ //表若不存在,则创建表
HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
HColumnDescriptor family = new HColumnDescriptor(family_name);
tableDescriptor.addFamily(family);
hbaseAdmin.createTable(tableDescriptor);
}
}
static class BatchImportMapper extends Mapper<LongWritable, Text, LongWritable, Text>{
Text v2 = new Text();
protected void map(LongWritable key, Text value, Context context) throws java.io.IOException ,InterruptedException {
final String[] splited = value.toString().split(" ");
try {
// System.out.println( splited[1]);
v2.set(value);
context.write(key, v2);
// System.out.println(key+” “+value);
// System.out.println(splited[0]);
} catch (NumberFormatException e) {
final Counter counter = context.getCounter(“BatchImport”, “ErrorFormat”);
counter.increment(1L);
System.out.println(“出错了”+splited[0]+” “+e.getMessage());
}
};
}
static class BatchImportReducer extends TableReducer<LongWritable, Text, NullWritable>{
@SuppressWarnings("deprecation")
protected void reduce(LongWritable key, java.lang.Iterable<Text> values, Context context) throws java.io.IOException ,InterruptedException {
for (Text text : values) {
final String[] splited = text.toString().split(" ");
String rowKey=splited[1];
System.out.println(splited[23]);
// System.out.println(“201309”+rowKey);
// System.out.println(splited[1]);
final Put put = new Put(Bytes.toBytes(rowKey)); //添加rowKey
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“the time of taking off”), Bytes.toBytes(splited[0].trim())); //添加列簇,列名,数值 预计起飞日期
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“飞机国籍登记号”), Bytes.toBytes(splited[1].trim())); //添加列簇,列名,数值
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“飞机类型”), Bytes.toBytes(splited[2].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“尾流等级”), Bytes.toBytes(splited[3].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“飞行计划类型”), Bytes.toBytes(splited[4].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“飞行种类”), Bytes.toBytes(splited[5].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“特殊机载设备”), Bytes.toBytes(splited[6].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“起飞机场”), Bytes.toBytes(splited[7].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“预计起飞时间”), Bytes.toBytes(splited[8].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“预计飞行时间”), Bytes.toBytes(splited[9].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“着陆机场”), Bytes.toBytes(splited[10].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“备降机场”), Bytes.toBytes(splited[11].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“真空速”), Bytes.toBytes(splited[12].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“预计巡航高度”), Bytes.toBytes(splited[13].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“飞行路径”), Bytes.toBytes(splited[14].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“驾驶员姓名”), Bytes.toBytes(splited[15].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“飞行证书号”), Bytes.toBytes(splited[16].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“驾驶员住址”), Bytes.toBytes(splited[17].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“电话号码”), Bytes.toBytes(splited[18].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“飞行基地”), Bytes.toBytes(splited[19].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“飞机载油量”), Bytes.toBytes(splited[20].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“登机人数”), Bytes.toBytes(splited[21].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“最多载人数”), Bytes.toBytes(splited[22].trim()));
put.add(Bytes.toBytes(“conf”), Bytes.toBytes(“飞机颜色”), Bytes.toBytes(splited[23].trim()));
//省略其他字段,调用put.add(….)即可
context.write(NullWritable.get(), put);
// System.out.println(NullWritable.get()+” “+put);
}
};
}
public static void main(String[] args) throws Exception {
hangxingbaowen.createTable();
final Configuration configuration = new Configuration();
//设置zookeeper
configuration.set("hbase.zookeeper.property.dataDir", "/home/liuchao/Hbase/zookeeper_data");
configuration.set("hbase.rootdir", "hdfs://localhost:9000/hbase");
configuration.set("hbase.zookeeper.quorum","localhost");
//设置hbase表名称
configuration.set(TableOutputFormat.OUTPUT_TABLE, "FPL");
//将该值改大,防止hbase超时退出
configuration.set("dfs.socket.timeout", "180000");
@SuppressWarnings("deprecation")
final Job job = new Job(configuration,"FPL");
job.setMapperClass(BatchImportMapper.class); //设置mapper工作任务
job.setReducerClass(BatchImportReducer.class); //设置reducer工作任务
//设置map的输出,不设置reduce的输出类型
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(Text.class);
job.setInputFormatClass(TextInputFormat.class);
//不再设置输出路径,而是设置输出格式类型
job.setOutputFormatClass(TableOutputFormat.class);
FileInputFormat.setInputPaths(job, "hdfs://Master:9000//usr/local/hadoop/飞行报文/FPL.txt");
job.waitForCompletion(true);
}
}