个人笔记,IBM合作的一个项目做了一个简单功能部分记录下。
package com.ibm.bi.ccb.TC_OLTP_06;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
import org.apache.hadoop.hbase.util.Bytes;
/**
* 操作HBASE工具类
*
* @author ZengWenfeng
* @date 2013-03-24
*/
public class HBaseUtil
{
/**
* 配置文件
*
* @author ZengWenfeng
* @date 2013-03-24
*/
private static Configuration conf = null;
/**
* 加载配置文件
*
* @author ZengWenfeng
* @date 2013-03-24
*/
static
{
conf = HBaseConfiguration.create();
// <configuration>
// <property>
// <name>hbase.cluster.distributed</name>
// <value>true</value>
// </property>
//
// <property>
// <name>hbase.rootdir</name>
// <value>hdfs://masternode:8020/hbase</value>
// </property>
//
// <property>
// <name>hbase.zookeeper.quorum</name>
// <value>masternode</value>
// </property>
// </configuration>
conf.set("hbase.cluster.distributed", "true");
conf.set("hbase.rootdir", "hdfs://masternode:8020/hbase");
conf.set("hbase.zookeeper.quorum", "slavenode6");
// conf.set("hbase.master", "gwnode1:8020");
// conf.set("hbase.zookeeper.quorum", "172.16.123.210");
// conf.set("hbase.zookeeper.property.clientPort", "8020");
}
public void loadFiles()
{
try
{
Path hfofDir = null;
HTable table = null;
LoadIncrementalHFiles loadFiles = new LoadIncrementalHFiles(conf);
loadFiles.doBulkLoad(hfofDir, table);
}
catch (Exception e)
{
e.printStackTrace();
}
}
/**
* 创建表
*
* @author ZengWenfeng
* @date 2013-03-24
* @param tabelNam 表名
* @param cfs 列族
* @throws IOException
*/
public void createTable(String tabelName, List<String> cfs) throws IOException
{
HBaseAdmin admin = new HBaseAdmin(conf);
if (admin.tableExists(tabelName))
{
System.out.println("table exist.");
}
else
{
HTableDescriptor tableDesc = new HTableDescriptor(tabelName);
if (cfs.isEmpty())
{
admin.close();
return;
}
for (int i = 0; i < cfs.size(); i++)
{
tableDesc.addFamily(new HColumnDescriptor(cfs.get(i)));
}
admin.createTable(tableDesc);
}
admin.close();
}
/**
* 删除表
*
* @author ZengWenfeng
* @date 2013-03-24
* @param tableName
* @throws IOException
*/
public void deleteTable(String tableName) throws IOException
{
HBaseAdmin admin = new HBaseAdmin(conf);
try
{
admin.disableTable(tableName);
admin.deleteTable(tableName);
System.out.println("删除成功");
}
catch (MasterNotRunningException e)
{
e.printStackTrace();
}
catch (ZooKeeperConnectionException e)
{
e.printStackTrace();
}
finally
{
admin.close();
}
}
/**
* 插入记录
*
* @author ZengWenfeng
* @date 2013-03-24
* @param tableName
* @param cfs
* @throws IOException
*/
public void writeRow(String tableName, List<String> cfs) throws IOException
{
HTable table = new HTable(conf, tableName);
try
{
Put put = new Put(Bytes.toBytes(cfs.get(0)));
put.add(Bytes.toBytes(cfs.get(1)), Bytes.toBytes(cfs.get(2)), Bytes.toBytes(cfs.get(3)));
table.put(put);
// table.put(puts);
System.out.println("插入成功");
}
catch (IOException e)
{
e.printStackTrace();
}
finally
{
table.close();
}
}
/**
* 更新记录
*
* @author ZengWenfeng
* @date 2013-03-24
* @param tableName
* @param cfs
* @throws IOException
*/
public void updateRow(String tableName, List<String> cfs) throws IOException
{
HTable table = new HTable(conf, tableName);
try
{
Put put = new Put(Bytes.toBytes(cfs.get(0)));
put.add(Bytes.toBytes(cfs.get(1)), Bytes.toBytes(cfs.get(2)), Bytes.toBytes(cfs.get(3)));
table.put(put);
System.out.println("success");
}
catch (IOException e)
{
e.printStackTrace();
}
finally
{
table.close();
}
}
/**
* 删除记录
*
* @author ZengWenfeng
* @date 2013-03-24
* @param tableName
* @param rowkey
* @throws IOException
*/
public void deleteRow(String tableName, String rowkey) throws IOException
{
HTable table = new HTable(conf, tableName);
List<Delete> list = new ArrayList<Delete>();
Delete d1 = new Delete(rowkey.getBytes());
list.add(d1);
table.delete(list);
System.out.println("删除成功");
table.close();
}
public void query(String tableName, String[] rowKeys) throws IOException
{
if (tableName == null || tableName.equals(tableName))
{
return;
}
}
/**
* 查询
*
* @param tableName
* @param rowkey
*/
public void selectRow(String tableName, String rowKey) throws IOException
{
HTable table = new HTable(conf, tableName);
Get g = new Get(rowKey.getBytes());
Result rs = table.get(g);
if (rs != null && rs.size() > 0)
{
for (KeyValue kv : rs.raw())
{
System.out.println(new String(kv.getRow()) + "|"
+ new String(kv.getFamily()) + ":"
+ new String(kv.getQualifier()) + "|"
+ new String(kv.getValue()) + "|" + kv.getTimestamp());
}
}
table.close();
}
/**
* 扫描表
*
* @param tableName
* @throws IOException
*/
public void scan(String tableName) throws IOException
{
HTable table = new HTable(conf, tableName);
try
{
Scan s = new Scan();
ResultScanner rs = table.getScanner(s);
for (Result r : rs)
{
KeyValue[] kv = r.raw();
for (int i = 0; i < kv.length; i++)
{
System.out.println(new String(kv[i].getRow()) + "|"
+ new String(kv[i].getFamily()) + ":"
+ new String(kv[i].getQualifier()) + "|"
+ new String(kv[i].getValue()) + "|"
+ kv[i].getTimestamp());
}
}
}
catch (IOException e)
{
e.printStackTrace();
}
finally
{
table.close();
}
}
/**
* 仅供测试
*
* @author ZengWenfeng
* @date 2013-03-24
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException
{
HBaseUtil hu = new HBaseUtil();
hu.deleteTable("zwf01");
// ArrayList<String> al = new ArrayList<String>();
// al.add("fdsafsd");
// al.add("bbb");
// hu.createTable("sfds", al);
// StringBuffer sb = new StringBuffer();
//
// for (String s : args)
// {
// sb.append(s + " ");
// }
//
// System.out.println(sb.toString());
//
// HBaseUtil hb = new HBaseUtil();
//
// if (args.length <= 1 || args[0].equalsIgnoreCase("help"))
// {
// System.out.println("--help--");
// }
// else if (args[0].equalsIgnoreCase("-c") || args[0].equalsIgnoreCase("create"))
// {
// List<String> strs = new ArrayList<String>();
//
// for (int i = 2; i < args.length; i++)
// {
// strs.add(args[i]);
// }
//
// hb.createTable(args[1], strs);
// }
// else if (args[0].equalsIgnoreCase("-dt") || args[0].equalsIgnoreCase("delete-table"))
// {
// hb.deleteTable(args[1]);
// }
// else if (args[0].equalsIgnoreCase("-i") || args[0].equalsIgnoreCase("insert"))
// {
// List<String> strs = new ArrayList<String>();
// for (int i = 2; i < args.length; i++)
// {
// strs.add(args[i]);
// }
//
// hb.writeRow(args[1], strs);
// }
// else if (args[0].equalsIgnoreCase("-dr") || args[0].equalsIgnoreCase("delete-row"))
// {
// hb.deleteRow(args[1], args[2]);
// }
// else if (args[0].equalsIgnoreCase("-qr") || args[0].equalsIgnoreCase("query-row"))
// {
// hb.selectRow(args[1], args[2]);
// }
// else if (args[0].equalsIgnoreCase("-qa") || args[0].equalsIgnoreCase("query-all"))
// {
// hb.scan(args[1]);
// }
// else if (args[0].equalsIgnoreCase("-u") || args[0].equalsIgnoreCase("update"))
// {
// List<String> strs = new ArrayList<String>();
//
// for (int i = 2; i < args.length; i++)
// {
// strs.add(args[i]);
// }
//
// hb.updateRow(args[1], strs);
// }
}
}
package com.ibm.bi.ccb.TC_OLTP_06;
import java.io.IOException;
/**
* @author ZengWenfeng
* @date 2013-04-11
*/
public class Task extends Thread
{
/**
* 表名
*/
private String tableName;
/**
* 主键
*/
private String[] rowKeys;
/**
* 构造函数
*/
public Task()
{
}
/**
* 构造函数
*/
public Task(String tableName, String[] rowKeys)
{
this.tableName = tableName;
this.rowKeys = rowKeys;
}
@Override
public void run()
{
System.out.println(Thread.currentThread().getName());
if (rowKeys != null && rowKeys.length > 0)
{
for (int i = 0; i < rowKeys.length; i++)
{
HBaseUtil hu = new HBaseUtil();
try
{
hu.selectRow(tableName, rowKeys[i]);
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
else
{
System.out.println("error");
}
}
}
package com.ibm.bi.ccb.TC_OLTP_06;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import com.ibm.bi.ccb.framework.ResourceManager;
/**
* TC_OLTP_06
*
* @author ZengWenfeng
* @date 2013-04-12
*/
public class TC_OLTP_06_Task extends Thread
{
/**
* 日志
*/
private static final Logger logger = ResourceManager.logger(TC_OLTP_06_Task.class);
/**
* 查询的表名
*/
private String tableName = "";
/**
* 多个主键组
*/
private String rowKeys = "";
/**
* 开始时间
*/
private long startTime = 0L;
/**
* 查询结果集合
*/
public static List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
/**
*
*/
public TC_OLTP_06_Task(long startTime)
{
this.startTime = startTime;
}
/**
* 根据给定的主键以及表名执行查询
*
* @see dev.toolkit.concurrent.Task#doTask()
*/
public void run()
{
try
{
ArrayList<Filter> al = new ArrayList<Filter>();
String[] keys = rowKeys.split(",");
for (int i = 0; i < keys.length; i++)
{
Filter filter = new RowFilter(CompareFilter.CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(keys[i])));
al.add(filter);
}
FilterList filterList = new FilterList(al);
Scan scan = new Scan();
scan.setFilter(filterList);
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.cluster.distributed", "true");
conf.set("hbase.rootdir", "hdfs://masternode:8020/hbase");
conf.set("hbase.zookeeper.quorum", "slavenode6");
HTable table = new HTable(conf, tableName);
ResultScanner rs = table.getScanner(scan);
list = new ArrayList<Map<String, Object>>();
logger.info("Scanning " + tableName + "...");
for (Result r : rs)
{
KeyValue[] kv = r.raw();
for (int i = 0; i < kv.length; i++)
{
Map<String, Object> data = new HashMap<String, Object>();
data.put("ROW", new String(kv[i].getRow()));
data.put("FAMILY_QUALIFIER", new String(kv[i].getFamily()) + ":" + new String(kv[i].getQualifier()));
data.put("VALUE", new String(kv[i].getValue()));
data.put("TIME", kv[i].getTimestamp());
list.add(data);
// System.out.println(new String(kv[i].getRow()) + "|"
// + new String(kv[i].getFamily()) + ":"
// + new String(kv[i].getQualifier()) + "|"
// + new String(kv[i].getValue()) + "|"
// + kv[i].getTimestamp());
}
}
rs.close();
long stopTime = System.currentTimeMillis();
logger.info("stopTime: " + stopTime + " ms");
long executeTime = stopTime - startTime;
logger.info("executeTime: " + executeTime + "ms");
}
catch (IOException e)
{
e.printStackTrace();
}
}
public String getTableName()
{
return tableName;
}
public void setTableName(String tableName)
{
this.tableName = tableName;
}
public String getRowKeys()
{
return rowKeys;
}
public void setRowKeys(String rowKeys)
{
this.rowKeys = rowKeys;
}
public long getStartTime()
{
return startTime;
}
public void setStartTime(long startTime)
{
this.startTime = startTime;
}
}
/**
* TC_OLTP_06.java
*
* 俗世翩翩少年歌一曲
* 把心声写给青山听
*/
package com.ibm.bi.ccb.TC_OLTP_06;
import java.util.ArrayList;
import org.slf4j.Logger;
import com.ibm.bi.ccb.framework.ResourceManager;
/**
* TC_OLTP_06
*
* @author ZengWenfeng
* @date 2013-04-12
*/
public class TC_OLTP_06
{
/**
* 日志
*/
private static final Logger logger = ResourceManager.logger(TC_OLTP_06.class);
/**
* 线程数
*/
private static int threadNumber = 1;
/**
* 任务数组
*/
private static TC_OLTP_06_Task[] tasks = null;
/**
* 构造函数
*/
public TC_OLTP_06()
{
logger.info("TC_OLTP_06 initialized");
}
/**
* 主函数,仅供测试
* wot, 1,12,2
*
* @author ZengWenfeng
* @date 2013-04-17
* @param args
*/
public static void main(String[] args)
{
//
String tableName = "";
String rowKeys = "";
// one argument
if (args.length == 1)
{
tableName = args[0];
}
// two arguments
else if (args.length == 2)
{
tableName = args[0];
rowKeys = args[1];
}
// exception
else
{
System.out.println("args is null.");
System.out.println("usage: Test <tableName> <rowKey1,rowKey2,...,rowKeyn>");
return;
}
//执行任务
excuteTask(tableName, rowKeys);
}
/**
* 执行任务
*
* @author ZengWenfeng
* @date 2013-04-17
* @param tableName
* @param rowKeys
*/
public static void excuteTask(String tableName, String rowKeys)
{
// 校验表名,为空不能执行查询
if (tableName == null || tableName.equals(""))
{
System.out.println("args[0] is table name. it is not null.");
System.out.println("usage: Test <tableName> <rowKey1,rowKey2,...,rowKeyn>");
return;
}
// 去重rowKeys
rowKeys = removeDuplicateRecords(rowKeys);
tasks = new TC_OLTP_06_Task[threadNumber];
try
{
long startTime = System.currentTimeMillis();
logger.info("startTime: " + startTime + " ms");
for (int i = 0; i < threadNumber; i++)
{
TC_OLTP_06_Task task = new TC_OLTP_06_Task(startTime);
task.setTableName(tableName);
task.setRowKeys(rowKeys);
task.start();
}
}
catch (Exception e)
{
e.printStackTrace();
}
}
/**
* 删除重复记录
*
* @author ZengWenfeng
* @date 2013-04-11
* @param rowKeys
* @return
*/
public static String removeDuplicateRecords(String rowKeys)
{
if (rowKeys.indexOf(",") <= 0)
{
return rowKeys;
}
String[] keys = rowKeys.split(",");
ArrayList<String> al = new ArrayList<String>();
//
for (int i = 0; i < keys.length; i++)
{
if (!al.contains(keys[i]))
{
al.add(keys[i]);
}
}
//
StringBuffer sb = new StringBuffer();
for (int j = 0; j < al.size(); j++)
{
sb.append(al.get(j));
if (j < al.size() - 1)
{
sb.append(",");
}
}
return sb.toString();
}
}
package com.ibm.bi.ccb.TC_OLTP_06;
import java.io.IOException;
import java.util.ArrayList;
import org.slf4j.Logger;
import com.ibm.bi.ccb.framework.ResourceManager;
/**
* TC_OLTP_06
*
* @author ZengWenfeng
* @date 2013-04-10
*/
public class Test
{
/**
* 日志
*/
private static final Logger logger = ResourceManager.logger(Test.class);
/**
* 最大线程数
*
* @author ZengWenfeng
* @date 2013-04-10
*/
private static int maxThreadNum = 30;
/**
* 构造函数
*
* @author ZengWenfeng
* @date 2013-04-10
*/
public Test()
{
logger.info("TC_OLTP_06 initialized");
}
/**
* 删除重复记录
*
* @author ZengWenfeng
* @date 2013-04-11
* @param rowKeys
* @return
*/
public static String removeDuplicateRecords(String rowKeys)
{
if (rowKeys.indexOf(",") <= 0)
{
return rowKeys;
}
String[] keys = rowKeys.split(",");
ArrayList<String> al = new ArrayList<String>();
//
for (int i = 0; i < keys.length; i++)
{
if (!al.contains(keys[i]))
{
al.add(keys[i]);
}
}
StringBuffer sb = new StringBuffer();
for (int j = 0; j < al.size(); j++)
{
sb.append(al.get(j));
if (j < al.size() - 1)
{
sb.append(",");
}
}
return sb.toString();
}
/**
* 仅供测试
*
* @author ZengWenfeng
* @date 2013-04-10
* @param args
*/
public static void main(String[] args)
{
//
String tableName = "";
String rowKeys = "";
// one argument
if (args.length == 1)
{
tableName = args[0];
}
// two arguments
else if (args.length == 2)
{
tableName = args[0];
rowKeys = args[1];
}
// exception
else
{
System.out.println("args is null.");
System.out.println("usage: Test <tableName> <rowKey1,rowKey2,...,rowKeyn>");
return;
}
// 校验表名,为空不能执行查询
if (tableName == null || tableName.equals(""))
{
System.out.println("args[0] is table name. it is not null.");
System.out.println("usage: Test <tableName> <rowKey1,rowKey2,...,rowKeyn>");
return;
}
// 去重rowKeys
rowKeys = removeDuplicateRecords(rowKeys);
// 当查询的row不为空
if (rowKeys != null && rowKeys.length() > 0)
{
//
String regex = ",";
int index = rowKeys.indexOf(regex);
// 查询条件的值为多个时
if (index > 0)
{
//
String[] keys = rowKeys.split(regex);
//
int allTasks = keys.length;
// 需要分配最大线程数
if (allTasks > maxThreadNum)
{
// 83 40 2 3
// 求余数3
int remainder = allTasks % maxThreadNum;
// 每个线程需要至少多少个任务 2
int tasksPerThread = allTasks / maxThreadNum;
// 根据最大线程数,启动多线程
for (int i = 0; i < maxThreadNum; i++)
{
// 00 01 02 ... 39
// 40 41 42 ... 79
// 80 81 82
int threadNum = 0;
if (i < remainder)
{
threadNum += tasksPerThread + 1;
}
else
{
threadNum += tasksPerThread;
}
String[] ids = new String[threadNum];
for (int j = 0; j < threadNum; j++)
{
ids[j] = keys[i + j * maxThreadNum];
}
Task t = new Task(tableName, ids);
t.start();
}
}
// 需要分配线程小于等于最大线程数
else
{
// 根据任务数,启动多线程
for (int i = 0; i < allTasks; i++)
{
String[] ids = new String[1];
ids[0] = keys[i];
Task t = new Task(tableName, ids);
t.start();
}
}
}
// 当查询条件值有且只有一个
else
{
HBaseUtil hbUtil = new HBaseUtil();
try
{
hbUtil.selectRow(tableName, rowKeys);
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
// 当查询条件为空,查询所有数据,执行scan 'tableName'
else
{
HBaseUtil hbUtil = new HBaseUtil();
try
{
hbUtil.scan(tableName);
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
}
package com.ibm.bi.ccb.TC_OLTP_06;
final class Timer
{
private static Long startTime;
private static Long endTime;
public static Long getStartTime()
{
return startTime;
}
public static void setStartTime(Long startTime)
{
Timer.startTime = startTime;
}
public static Long getEndTime()
{
return endTime;
}
public static void setEndTime(Long endTime)
{
Timer.endTime = endTime;
}
public static long stopTimerAndGetElapseTime()
{
Long startTime =getStartTime();
Long endTime = System.currentTimeMillis();
// System.out.println("End time: " + endTime +
// " for current thread id: "
// + Thread.currentThread().getId());
if (startTime == null)
{
return -1;
}
else
{
return endTime - startTime;
}
}
}