hadoop和hive的安装与初步使用

hadoop安装参考 http://www.powerxing.com/install-hadoop/

hive 安装配置参考 http://www.mincoder.com/article/5809.shtml

                               http://blog.csdn.net/reesun/article/details/8556078

安装完成后本地运行设置好相应的配置文件即可。首先启动Hadoop，$HADOOP_HOME/sbin/start-dfs.sh， 用jps查看进程状况，需要看见NameNode和DataNode， 然后执行${HIVE_HOME}/bin/hive --service metastore 启动元数据， 执行${HIVE_HOME}/bin/hiveserver2 运行服务端服务，可hive进入命令行执行相关命令也可以用spring-hivejdbc代码的形式访问。

public class HiveQuery {
    @Resource
    JdbcTemplate hiveJdbcTemplate;

    private List<Map<String, Object>> ret = Lists.newArrayList();

    RowCallbackHandler handler = new RowCallbackHandler() {
        @Override
        public void processRow(ResultSet rs) throws SQLException {
            ResultSetMetaData rm = rs.getMetaData();
            int colName = rm.getColumnCount();
            Map<String, Object> map = new HashMap();
            for (int i = 1; i <= colName; i++) {
                map.put(rm.getColumnName(i), rs.getObject(i));
            }
            ret.add(map);
        }
    };

    public boolean preparePartition(String tableName, String pt, String hdfspath) {
        if (StringUtils.isEmpty(tableName)) return false;
        if (StringUtils.isEmpty(pt)) return false;
        if (StringUtils.isEmpty(hdfspath)) return false;

        String hql = "ALTER table %s add  IF NOT EXISTS partition(pt=%s) location %s";
        hiveJdbcTemplate.execute(String.format(hql, tableName, pt, hdfspath));
        return true;
    }

    public boolean prepareDW(String tableName, String pt) {
        String hdfs = "/user/hive/warehouse/%s/%s.json";
        return preparePartition(tableName, pt, String.format(hdfs, tableName, pt));
    }

    public void execHQLWithoutRet(String hql) {
        hiveJdbcTemplate.execute(hql);
    }

    public void exeHQL(String hql) {
        hiveJdbcTemplate.query(hql, handler);
    }

    public void exeBatch(final List<String> hqls) {
        hiveJdbcTemplate.execute(new StatementCallback<Object>() {
            @Override
            public Object doInStatement(Statement stmt) throws SQLException, DataAccessException {
                for (String hql : hqls) {
                    if(isSelectable(hql)){
                        ResultSet rs =stmt.executeQuery(hql);
                        while(rs.next()) handler.processRow(rs);
                    }else{
                        stmt.execute(hql);
                    }
                }
                return null;
            }
        });
    }


    public HiveQuery() {
        this.hiveJdbcTemplate = (JdbcTemplate) CtxHolder.obtainBean("hiveJdbcTemplate");
    }


    private boolean isSelectable(String sql) {
        return sql.trim().startsWith("select");
    }


    public List<Map<String, Object>> fetchResult() {
        return ret;
    }
}