HbaseNewUtils

package com.youku.laifeng;

import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.security.UserGroupInformation;

public class HbaseNewUtils {
	private Configuration conf = null;
	HTable table = null;
	private String tableName;

	public HbaseNewUtils(PropertiesType propertiesType, String tableName) {
		conf = HBaseConfiguration.create();
		try {
			// PropertiesUtil properties = new
			// PropertiesUtil(propertiesType.getValue());
			conf.set("hbase.zookeeper.quorum", "20.100.25.39,20.100.25.40,20.100.25.41");
			conf.set("hbase.zookeeper.property.clientPort", "2182");
			conf.set("hbase.master.kerberos.principal", "hbase/_HOST@HBASE.YOUKU");
			conf.set("hbase.regionserver.kerberos.principal", "hbase/_HOST@HBASE.YOUKU");
			conf.set("hbase.security.authentication", "kerberos");
			conf.set("hadoop.security.authentication", "kerberos");
			conf.set("hbase.rpc.protection", "privacy");

			UserGroupInformation.setConfiguration(conf);
			UserGroupInformation.loginUserFromKeytab("yule@HBASE.YOUKU", "/home/cafe/.keys/yule1.keytab");
			table = new HTable(conf, Bytes.toBytes(tableName));
		} catch (Exception e) {
		}

		this.tableName = tableName;
	}

	/**
	 * 添加数据
	 * 
	 * @param rowKey
	 * @param column
	 * @param value
	 */
	public void addData(String rowKey, String column, Object value) {
		try {
			Put put = new Put(Bytes.toBytes(rowKey));
			put.add(Bytes.toBytes(column), Bytes.toBytes(column), Bytes.toBytes(value.toString()));
			table.put(put);
		} catch (IOException e) {
		}
	}

	/**
	 * 批量添加数据
	 * 
	 * @param list
	 */
	public void addDataBatch(List<Put> list) {
		try {
			table.put(list);
		} catch (RetriesExhaustedWithDetailsException e) {
		} catch (InterruptedIOException e) {
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	/**
	 * 创建�?
	 * 
	 * @param tableName
	 *            表名
	 * @param family
	 *            列名
	 * @throws Exception
	 */
	public void creatTable(String tableName, String[] family) throws Exception {
		HBaseAdmin admin = new HBaseAdmin(conf);
		HTableDescriptor desc = new HTableDescriptor(tableName);
		for (int i = 0; i < family.length; i++) {
			HColumnDescriptor columnDesc = new HColumnDescriptor(family[i]);
			desc.addFamily(columnDesc);
		}
		if (admin.tableExists(tableName)) {
			System.out.println("table Exists!");
			System.exit(0);
		} else {
			admin.createTable(desc);
			System.out.println("create table Success!");
		}
	}

	public void dropTable() {
		try {
			HBaseAdmin admin = new HBaseAdmin(conf);
			admin.disableTable(tableName);
			admin.deleteTable(tableName);
		} catch (MasterNotRunningException e) {
			e.printStackTrace();
		} catch (ZooKeeperConnectionException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}

	}

	/**
	 * 查询全部
	 */
	public void queryAll() {
		Scan scan = new Scan();
		try {
			ResultScanner results = table.getScanner(scan);
			for (Result result : results) {
				int i = 0;
				for (KeyValue rowKV : result.list()) {

					if (i++ == 0) {
						System.out.print("rowkey:" + new String(rowKV.getRow()) + " ");
					}
					System.out.print(" " + new String(rowKV.getQualifier()) + " ");
					System.out.print(":" + new String(rowKV.getValue()));

					if (i > 100) {
						return;
					}
				}

				System.out.println();
			}
		} catch (IOException e) {
		}

	}

	/**
	 * 按某字段查询 column = value 的数�?
	 * 
	 * @param queryColumn
	 *            要查询的列名
	 * @param value
	 *            过滤条件�?
	 * @param columns
	 *            返回的列名集�?
	 */
	public ResultScanner queryBySingleColumn(String queryColumn, String value, String[] columns) {
		if (columns == null || queryColumn == null || value == null) {
			return null;
		}

		try {
			SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes(queryColumn), Bytes.toBytes(queryColumn), CompareOp.EQUAL, new SubstringComparator(value));
			Scan scan = new Scan();

			for (String columnName : columns) {
				scan.addColumn(Bytes.toBytes(columnName), Bytes.toBytes(columnName));
			}

			Calendar calendar = Calendar.getInstance();
			calendar.set(Calendar.HOUR_OF_DAY, 0);
			calendar.set(Calendar.MINUTE, 0);
			calendar.set(Calendar.SECOND, 0);

			long startTimestap = calendar.getTimeInMillis();

			Calendar calendar2 = Calendar.getInstance();
			long endTimestap = calendar2.getTimeInMillis();

			Date sdate = new Date(startTimestap);
			Date edate = new Date(endTimestap);

			System.out.println(sdate + "\t" + edate);
			System.out.println(startTimestap + "\t" + endTimestap);

			scan.setFilter(filter);
			scan.setTimeRange(startTimestap, endTimestap);
			return table.getScanner(scan);
		} catch (Exception e) {
		}

		return null;
	}

	/**
	 * 根据对比条件，按某字段查�? column = value 的数�?
	 * 
	 * @param queryColumn
	 *            要查询的列名
	 * @param value
	 *            过滤条件�?
	 * @param columns
	 *            返回的列名集�?
	 * @param op
	 *            操作符号 CompareOp.EQUAL 代表等于,CompareOp.LESS代表小于
	 */
	public ResultScanner queryBySingleColumnOnCompareOp(String queryColumn, String value, String[] columns, CompareOp op) {
		if (columns == null || queryColumn == null || value == null) {
			return null;
		}

		try {
			SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes(queryColumn), Bytes.toBytes(queryColumn), op, new SubstringComparator(value));
			Scan scan = new Scan();

			for (String columnName : columns) {
				scan.addColumn(Bytes.toBytes(columnName), Bytes.toBytes(columnName));
			}

			scan.setFilter(filter);
			return table.getScanner(scan);
		} catch (Exception e) {
		}

		return null;
	}

	/**
	 * 在指定的条件下，按某�?字段聚合
	 * 
	 * @param paramMap
	 *            参数条件
	 * @param dimensionColumns
	 *            维度
	 * @param aggregateColumn
	 *            聚合字段
	 * @return 返回map，key 为dimensionColumns 维度相对应的数据，value 为aggregateColumn
	 *         字段对应的�??
	 */
	public Map<String, Long> aggregateBySingleColumn(Map<String, String> paramMap, String[] dimensionColumns, String aggregateColumn) {
		if (dimensionColumns == null || dimensionColumns.length == 0 || paramMap == null || aggregateColumn == null || aggregateColumn.equals("")) {
			return null;
		}

		Map<String, Long> map = null;
		try {
			FilterList filterList = new FilterList();
			Scan scan = new Scan();
			// 添加过滤条件
			for (String paramKey : paramMap.keySet()) {
				SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes(paramKey), Bytes.toBytes(paramKey), CompareOp.EQUAL, new SubstringComparator(paramMap.get(paramKey)));
				filterList.addFilter(filter);
			}
			scan.setFilter(filterList);

			// 要展现的�?
			for (String column : dimensionColumns) {
				scan.addColumn(Bytes.toBytes(column), Bytes.toBytes(column));
			}
			scan.addColumn(Bytes.toBytes(aggregateColumn), Bytes.toBytes(aggregateColumn));

			ResultScanner results = table.getScanner(scan);

			// 将查询结果放入map �?
			map = new ConcurrentHashMap<String, Long>();
			for (Result result : results) {
				// String dimensionKey = "";
				StringBuilder dimensionKey = new StringBuilder();
				// 取�??
				String value = new String(result.getValue(Bytes.toBytes(aggregateColumn), Bytes.toBytes(aggregateColumn)));
				Long aggregateValue = value == null ? 0 : Long.parseLong(value);

				// 拼接Key
				for (String column : dimensionColumns) {
					dimensionKey.append("\t" + new String(result.getValue(Bytes.toBytes(column), Bytes.toBytes(column))));
				}
				dimensionKey = dimensionKey.deleteCharAt(0);

				if (map.containsKey(dimensionKey)) {
					map.put(dimensionKey.toString(), map.get(dimensionKey.toString()) + aggregateValue);
				} else {
					map.put(dimensionKey.toString(), aggregateValue);
				}
			}
		} catch (Exception e) {
		}

		return map;
	}

	public static void main(String[] args) throws Exception {
		String tableName = "lf:lf_t_result_room_online_min_user_stat";

		if (args != null && args.length > 0) {
			tableName = args[0];
		}

		HbaseNewUtils util = new HbaseNewUtils(PropertiesType.DDSHOW_HASE, tableName);

		ResultScanner results = util.queryBySingleColumn("stat_date", "2016-03-17", new String[] { "stat_date", "min", "uv" });

		Map<String, Integer> map = new HashMap<String, Integer>();
		for (Result result : results) {
			int i = 0;

			String min = "";
		    int uv = 0;

			for (KeyValue rowKV : result.list()) {
				String key = new String(rowKV.getQualifier());
				String value = new String(rowKV.getValue());
				if(key.equals("min")) {
					min = value;
				}

				if(key.equals("uv")) {
					if(value != null && !value.equals("")) {
						uv = Integer.parseInt(value);
					}
				}
			}

			if(map.containsKey(min)) {
				map.put(min, map.get(min) + uv);
			} else {
				map.put(min, uv);
			}
		}

		for(String key : map.keySet()) {
			System.out.println(key + "\t" + map.get(key));
		}
	}
}