package cn.yc.com.base.data;
/**
*
* ClassName: RecordBean
* @Description: 实时访客Hbase表映射
* @author 余辉
* @date 2015-8-18
*/
public class RecordBean {
private String rowkey; //rowkey
private String visitTime; //访问时间
private String source; //来源
private String visitPage; //访问页面
private String ip; //访客ip
private String traceCode; //访客标识
/**
* 返回json格式
* @return
*/
public String toJson(){
return "{'"+"rowkey"+"':'"+getRowkey()+"'"+",'"+"visitTime"+"':'"+getVisitTime()+"'"+",'"+"source"+"':'"+getSource()+"'"+",'"+"visitPage"+"':'"+getVisitPage()+"'"+",'"+"ip"+"':'"+getIp()+"'"+",'"+"traceCode"+"':'"+getTraceCode()+"}";
}
/***************************************************/
public String getVisitTime() {
return visitTime;
}
public void setVisitTime(String visitTime) {
this.visitTime = visitTime;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public String getVisitPage() {
return visitPage;
}
public void setVisitPage(String visitPage) {
this.visitPage = visitPage;
}
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
public String getTraceCode() {
return traceCode;
}
public void setTraceCode(String traceCode) {
this.traceCode = traceCode;
}
public String getRowkey() {
return rowkey;
}
public void setRowkey(String rowkey) {
this.rowkey = rowkey;
}
}
package cn.yc.com.util;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.RegionSplitter;
import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm;
import cn.yc.com.base.data.RecordBean;
/**
*
* ClassName: HbaseUtil
* @Description: Hbase交互工具类
* @author 余辉
* @date 2015-8-18
*/
public class HbaseDao {
public static Configuration configuration;
static {
configuration = HBaseConfiguration.create();
}
/**
* 创建表
* @return boolean
* @throws Exception
*/
public boolean createTable(String tableName,String[] familys,Integer regionNumber) throws Exception {
HBaseAdmin hBaseAdmin = new HBaseAdmin(configuration);
if(checkTableExists(tableName)){
dropTable(hBaseAdmin,tableName);
}
HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
if(familys!=null){
for (String familyName : familys) {
HColumnDescriptor hc=new HColumnDescriptor(familyName);
hc.setBlockCacheEnabled(true);
hc.setBloomFilterType(BloomType.ROWCOL);
hc.setCompressionType(Compression.Algorithm.SNAPPY);
hc.setMaxVersions(1);
tableDescriptor.addFamily(hc);
}
}
Configuration conf = HBaseConfiguration.create();
SplitAlgorithm splitter = RegionSplitter.newSplitAlgoInstance(conf,"HexStringSplit");
if(!hBaseAdmin.tableExists(tableName)){
hBaseAdmin.createTable(tableDescriptor, splitter.split(regionNumber));
}
closeLink(null,null,hBaseAdmin);
return checkTableExists(tableName);
}
/**
* 检查指定表是否存在
* @return boolean
*/
public boolean checkTableExists(String tableName){
HBaseAdmin hBaseAdmin = null;
try {
hBaseAdmin = new HBaseAdmin(configuration);
} catch (MasterNotRunningException e) {
e.printStackTrace();
} catch (ZooKeeperConnectionException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
try {
if(hBaseAdmin.tableExists(tableName)) {
return true;
}
} catch (IOException e) {
e.printStackTrace();
}finally{
closeLink(null,null,hBaseAdmin);
}
return false;
}
/**
* 插入一条数据
*/
public void insertData(String tableName,String rowkey,String family,String rowName,String value) {
HConnection connection = null;
HTable table = null;
try {
connection = HConnectionManager.createConnection(configuration);
table = (HTable) connection.getTable(tableName);
} catch (IOException e1) {
e1.printStackTrace();
}
Put put = new Put(rowkey.getBytes());
put.add(family.getBytes(), rowName.getBytes(), value.getBytes());
try {
table.put(put);
} catch (IOException e) {
e.printStackTrace();
}finally{
closeLink(table,connection,null);
}
}
/**
* 插入一行数据
* @return boolean
*/
public boolean insertData(String tableName,String family,Map dataMap) {
boolean status = true;
String rowkey = (String) dataMap.get("rowkey");
HConnection connection = null;
HTable table = null;
try {
connection = HConnectionManager.createConnection(configuration);
table = (HTable) connection.getTable(tableName);
} catch (IOException e1) {
e1.printStackTrace();
}
Put put = new Put(rowkey.getBytes());
Set<Map.Entry<String, String>> set = dataMap.entrySet();
for (Iterator<Map.Entry<String, String>> it = set.iterator(); it.hasNext();) {
Map.Entry<String, String> entry = (Map.Entry<String, String>) it.next();
if(!entry.getKey().equals("rowkey")){
put.add(family.getBytes(),entry.getKey().getBytes(),entry.getValue().getBytes());
}
}
try {
table.put(put);
} catch (IOException e) {
status = false;
e.printStackTrace();
}finally{
closeLink(table,connection,null);
}
return status;
}
/**
* 删除指定表
* @return boolean
*/
@SuppressWarnings("finally")
public boolean dropTable(HBaseAdmin admin,String tableName){
try {
if(admin == null){
admin = new HBaseAdmin(configuration);
}
admin.disableTable(tableName);
admin.deleteTable(tableName);
} catch (MasterNotRunningException e) {
e.printStackTrace();
} catch (ZooKeeperConnectionException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}finally{
try {
admin.close();
} catch (IOException e){
e.printStackTrace();
}
return checkTableExists(tableName);
}
}
/**
* 根据rowkeyAndTableName删除指定行
*/
public void deleteRowByRowkeyAndTableName(String tablename, String rowkey) {
HConnection connection = null;
HTable table = null;
try {
connection = HConnectionManager.createConnection(configuration);
table = (HTable) connection.getTable(tablename);
List list = new ArrayList();
Delete d1 = new Delete(rowkey.getBytes());
list.add(d1);
table.delete(list);
} catch (IOException e1) {
e1.printStackTrace();
}finally{
closeLink(table,connection,null);
}
}
/**
* 根据rowkey查询一条记录
* @return resultMap
*/
@SuppressWarnings({ "finally", "deprecation" })
public Map getMapByRowkeyAndTableName(String tableName,String rowkey) {
Map resultMap = null;
HConnection connection = null;
HTable table = null;
try {
connection = HConnectionManager.createConnection(configuration);
table = (HTable) connection.getTable(tableName);
Get scan = new Get(rowkey.getBytes());
Result r = table.get(scan);
resultMap = new HashMap<String,Object>();
if(null == r || r.isEmpty()){
return null;
}
for (KeyValue keyValue : r.raw()) {
String key = Bytes.toString(keyValue.getFamily())+":"+Bytes.toString(keyValue.getQualifier());
resultMap.put(key,Bytes.toString(keyValue.getValue()));
}
} catch (IOException e1) {
//e1.printStackTrace();
}finally{
closeLink(table,connection,null);
return resultMap;
}
}
/**
* 单条件查询多条记录 Filter
* 查询指定列族family的值等于value的记录
*/
@SuppressWarnings("finally")
public List<HashMap<String, Object>> QueryByCondition2(String tableName,String family,String column,String value) {
List<HashMap<String, Object>> results = null;
HConnection connection = null;
HTable table = null;
try {
connection = HConnectionManager.createConnection(configuration);
table = (HTable) connection.getTable(tableName);
Filter filter = new SingleColumnValueFilter(
Bytes.toBytes(family), Bytes.toBytes(column), CompareOp.EQUAL,
new SubstringComparator(value));
Scan s = new Scan();
s.setFilter(filter);
ResultScanner rs = table.getScanner(s);
results = new ArrayList<HashMap<String,Object>>();
for (Result r : rs) {
HashMap<String, Object> map = new HashMap<String, Object>();
for (KeyValue keyValue : r.raw()) {
String key = keyValue.getFamily()+":"+keyValue.getQualifier();
map.put(key,keyValue.getValue());
}
results.add(map);
}
rs.close();
} catch (IOException e1) {
e1.printStackTrace();
}finally{
closeLink(table,connection,null);
return results;
}
}
/**
* 根据一组rowkey和表名获取一组RecordBean
* @return records
*/
public List<RecordBean> getRecordByRowkeyAndTableName(String tableName,List<String> rowkeys) {
List<RecordBean> records = null;
HConnection connection = null;
HTable table = null;
List<Get> list = new ArrayList<Get>();
Get get = null;
for(int i=0;i<rowkeys.size();i++){
get = new Get(Bytes.toBytes(rowkeys.get(i)));
list.add(get);
}
try{
connection = HConnectionManager.createConnection(configuration);
table = (HTable) connection.getTable(tableName);
Result[] res = table.get(list);
records = new ArrayList<RecordBean>();
byte[] rowkey = null;
byte[] visitTime = null;
byte[] source = null;
byte[] visitPage = null;
byte[] ip = null;
byte[] traceCode = null;
for (Result rs : res) {
RecordBean record = new RecordBean();
rowkey = rs.getRow();
if(null != rowkey && 0<rowkey.length){
record.setRowkey(new String(rowkey));
}
visitTime = rs.getValue("cs".getBytes(), "visitTime".getBytes());
if(null != visitTime && 0<visitTime.length){
record.setVisitTime(new String(visitTime));
}
ip = rs.getValue("cs".getBytes(), "IP".getBytes());
if(null != ip && 0<ip.length){
record.setIp(new String(ip));
}
traceCode = rs.getValue("cs".getBytes(), "traceCode".getBytes());
if(null != traceCode && 0<traceCode.length){
record.setTraceCode(new String(traceCode));
}
visitPage = rs.getValue("cs".getBytes(), "visitPage".getBytes());
if(null != visitPage && 0<visitPage.length){
record.setVisitPage(new String(visitPage));
}
source = rs.getValue("cs".getBytes(), "source".getBytes());
if(null != source && 0<source.length){
record.setSource(new String(source));
}
records.add(record);
}
}catch (Exception e) {
e.printStackTrace();
}finally{
closeLink(table,connection,null);
}
return records;
}
/**
* 根据rowkey和表面获取一条 RecordBean
* @param tableName,rowkey
* @return record
*/
public RecordBean getRecordByRowkeyAndTableName(String tableName,String rowkeyT) {
HConnection connection = null;
HTable table = null;
RecordBean record = new RecordBean();
Get get = new Get(Bytes.toBytes(rowkeyT));
try{
connection = HConnectionManager.createConnection(configuration);
table = (HTable) connection.getTable(tableName);
Result rs = table.get(get);
byte[] rowkey = null;
byte[] visitTime = null;
byte[] source = null;
byte[] visitPage = null;
byte[] ip = null;
byte[] traceCode = null;
rowkey = rs.getRow();
if(null != rowkey && 0<rowkey.length){
record.setRowkey(new String(rowkey));
}
visitTime = rs.getValue("cs".getBytes(), "visitTime".getBytes());
if(null != visitTime && 0<visitTime.length){
record.setVisitTime(new String(visitTime));
}
ip = rs.getValue("cs".getBytes(), "IP".getBytes());
if(null != ip && 0<ip.length){
record.setIp(new String(ip));
}
traceCode = rs.getValue("cs".getBytes(), "traceCode".getBytes());
if(null != traceCode && 0<traceCode.length){
record.setTraceCode(new String(traceCode));
}
visitPage = rs.getValue("cs".getBytes(), "visitPage".getBytes());
if(null != visitPage && 0<visitPage.length){
record.setVisitPage(new String(visitPage));
}
source = rs.getValue("cs".getBytes(), "source".getBytes());
if(null != source && 0<source.length){
record.setSource(new String(source));
}
}catch (Exception e) {
e.printStackTrace();
}finally{
closeLink(table,connection,null);
}
return record;
}
public List<RecordBean> getListRecord(String tableName,String prefix,String Num) {
List<RecordBean> records = null;
HConnection connection = null;
HTable table = null;
ResultScanner rse = null;
Scan scan = new Scan();
scan.setCaching(500);
scan.addFamily("cs".getBytes());
String rowk = prefix;
scan.setFilter(new PageFilter(Long.parseLong(Num)));
scan.setStartRow(rowk.getBytes());
try{
connection = HConnectionManager.createConnection(configuration);
table = (HTable) connection.getTable(tableName);
rse = table.getScanner(scan);
byte[] rowkey = null;
byte[] visitTime = null;
byte[] source = null;
byte[] visitPage = null;
byte[] ip = null;
byte[] traceCode = null;
records = new ArrayList<RecordBean>();
for (Result rs : rse) {
RecordBean record = new RecordBean();
rowkey = rs.getRow();
if(null != rowkey && 0<rowkey.length){
record.setRowkey(new String(rowkey));
}
visitTime = rs.getValue("cs".getBytes(), "visitTime".getBytes());
if(null != visitTime && 0<visitTime.length){
record.setVisitTime(new String(visitTime));
}
ip = rs.getValue("cs".getBytes(), "IP".getBytes());
if(null != ip && 0<ip.length){
record.setIp(new String(ip));
}
traceCode = rs.getValue("cs".getBytes(), "traceCode".getBytes());
if(null != traceCode && 0<traceCode.length){
record.setTraceCode(new String(traceCode));
}
visitPage = rs.getValue("cs".getBytes(), "visitPage".getBytes());
if(null != visitPage && 0<visitPage.length){
record.setVisitPage(new String(visitPage));
}
source = rs.getValue("cs".getBytes(), "source".getBytes());
if(null != source && 0<source.length){
record.setSource(new String(source));
}
records.add(record);
}
}catch (Exception e) {
e.printStackTrace();
}finally{
closeLink(table,connection,null);
}
return records;
}
/**
* 关闭连接
* @param table
* @param conn
* @param admin
*/
public void closeLink(HTable table,HConnection conn,HBaseAdmin admin){
try {
if(table!=null){
table.close();
}if(conn!=null){
conn.close();
}if(admin!=null){
admin.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
北京小辉微信公众号
大数据资料分享请关注
扫一扫
579
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
