java flume hbase_应用Flume+HBase采集和存储日志数据

packagecom.ncc.dlut;importjava.io.UnsupportedEncodingException;importjava.util.ArrayList;importjava.util.List;importorg.apache.flume.Context;importorg.apache.flume.Event;importorg.apache.flume.conf.ComponentConfiguration;importorg.apache.flume.sink.hbase.AsyncHbaseEventSerializer;importorg.apache.flume.sink.hbase.SimpleRowKeyGenerator;importorg.hbase.async.AtomicIncrementRequest;importorg.hbase.async.PutRequest;public class AsyncHbaseLTEEventSerializer implementsAsyncHbaseEventSerializer {//表名

private byte[] table;//列族

private byte[] colFam;//当前事件

privateEvent currentEvent;//列名

private byte[][] columnNames;//用于向HBase批量存储数据

private final List puts = new ArrayList();private final List incs = new ArrayList();//当前行键

private byte[] currentRowKey;private final byte[] eventCountCol = "eventCount".getBytes();

@Overridepublic voidconfigure(Context context) {//从配置文件中获取列名

String cols = new String(context.getString("columns"));

String[] names= cols.split(",");

columnNames= new byte[names.length][];int i = 0;for(String name:names){

columnNames[i++] =name.getBytes();

}

}

@Overridepublic voidconfigure(ComponentConfiguration conf) {//TODO Auto-generated method stub

}

@Overridepublic voidcleanUp() {//TODO Auto-generated method stub

table = null;

colFam= null;

currentEvent= null;

columnNames= null;

currentRowKey= null;

}

@Overridepublic ListgetActions() {//分割事件体获取各列的值

String eventStr = newString(currentEvent.getBody());

String[] cols=logTokenize(eventStr);

puts.clear();//数据中的时间

String time=cols[1];int n1 = 13-time.length();

StringBuilder sb= newStringBuilder(time);for(int i=0;i

sb.insert(0, '0');

}try{//使用自带的行键生成器生成行键

currentRowKey = SimpleRowKeyGenerator.getUUIDKey(cols[0]+"-"+sb.toString());

}catch(UnsupportedEncodingException e) {//TODO Auto-generated catch block

e.printStackTrace();

}//currentRowKey = (cols[0]+"-"+System.currentTimeMillis()).getBytes();

int n =cols.length;//添加每列数据

for(int i=0;i

PutRequest putReq= newPutRequest(table, currentRowKey,colFam,columnNames[i],cols[i].getBytes());

puts.add(putReq);

}returnputs;

}

@Overridepublic ListgetIncrements() {//增加接收到的事件数量

incs.clear();

incs.add(new AtomicIncrementRequest(table, "totalEvents".getBytes(), colFam, eventCountCol));returnincs;

}

@Override//初始化表名和列名

public void initialize(byte[] table, byte[] cf) {this.table =table;this.colFam =cf;

}

@Overridepublic voidsetEvent(Event event) {//TODO Auto-generated method stub

this.currentEvent =event;

}//从日志中获取列值信息

publicString[] logTokenize(String eventStr) {//String logEntryPattern = "^([\\d.]+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(.+?)\" (\\d{3}) (\\d+|-) \"([^\"]+)\" \"([^\"]+)\"";//Pattern p = Pattern.compile(logEntryPattern);//Matcher matcher = p.matcher(eventStr);

/*if (!matcher.matches()){

System.err.println("Bad log entry (or problem with RE?):");

System.err.println(eventStr);

return null;

}*/

/*String[] columns = new String[matcher.groupCount()];

for (int i = 0; i < matcher.groupCount(); i++){

columns[i] = matcher.group(i+1);

}*/String[] s= eventStr.split("[:,]");int n =s.length;

String[] columns= new String[n/2];for(int i=0;2*i+1

columns[i]= s[2*i+1];

}returncolumns;

}

}