hadoop安装参考 http://www.powerxing.com/install-hadoop/
hive 安装配置参考 http://www.mincoder.com/article/5809.shtml
http://blog.csdn.net/reesun/article/details/8556078
安装完成后本地运行设置好相应的配置文件即可。首先启动Hadoop,$HADOOP_HOME/sbin/start-dfs.sh, 用jps查看进程状况,需要看见NameNode和DataNode, 然后执行${HIVE_HOME}/bin/hive --service metastore 启动元数据, 执行${HIVE_HOME}/bin/hiveserver2 运行服务端服务,可hive进入命令行执行相关命令也可以用spring-hivejdbc代码的形式访问。
public class HiveQuery {
@Resource
JdbcTemplate hiveJdbcTemplate;
private List<Map<String, Object>> ret = Lists.newArrayList();
RowCallbackHandler handler = new RowCallbackHandler() {
@Override
public void processRow(ResultSet rs) throws SQLException {
ResultSetMetaData rm = rs.getMetaData();
int colName = rm.getColumnCount();
Map<String, Object> map = new HashMap();
for (int i = 1; i <= colName; i++) {
map.put(rm.getColumnName(i), rs.getObject(i));
}
ret.add(map);
}
};
public boolean preparePartition(String tableName, String pt, String hdfspath) {
if (StringUtils.isEmpty(tableName)) return false;
if (StringUtils.isEmpty(pt)) return false;
if (StringUtils.isEmpty(hdfspath)) return false;
String hql = "ALTER table %s add IF NOT EXISTS partition(pt=%s) location %s";
hiveJdbcTemplate.execute(String.format(hql, tableName, pt, hdfspath));
return true;
}
public boolean prepareDW(String tableName, String pt) {
String hdfs = "/user/hive/warehouse/%s/%s.json";
return preparePartition(tableName, pt, String.format(hdfs, tableName, pt));
}
public void execHQLWithoutRet(String hql) {
hiveJdbcTemplate.execute(hql);
}
public void exeHQL(String hql) {
hiveJdbcTemplate.query(hql, handler);
}
public void exeBatch(final List<String> hqls) {
hiveJdbcTemplate.execute(new StatementCallback<Object>() {
@Override
public Object doInStatement(Statement stmt) throws SQLException, DataAccessException {
for (String hql : hqls) {
if(isSelectable(hql)){
ResultSet rs =stmt.executeQuery(hql);
while(rs.next()) handler.processRow(rs);
}else{
stmt.execute(hql);
}
}
return null;
}
});
}
public HiveQuery() {
this.hiveJdbcTemplate = (JdbcTemplate) CtxHolder.obtainBean("hiveJdbcTemplate");
}
private boolean isSelectable(String sql) {
return sql.trim().startsWith("select");
}
public List<Map<String, Object>> fetchResult() {
return ret;
}
}