dw_get_bribery

import java.io.IOException;
   import java.io.Serializable;
import java.util.List;
import java.util.Date;
import DAO.Conn;
import DAO.ScalaConn;
import DAO.SparkHBase;
import org.apache.commons.lang3.time.DateFormatUtils;
    import org.apache.hadoop.conf.Configuration;
   import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Table;
import org.apache.spark.sql.Dataset;
   import org.apache.spark.sql.Row;
import org.slf4j.Logger;
   import org.slf4j.LoggerFactory;

import java.sql.Timestamp;

/*
* 平台获得红包数据。
* author：谭志坚
* */

public class dw_get_bribery implements Serializable {
   private static final long serialVersionUID = -1;
   // private static final long serialVersionUID= 6732270565076291202;
   private static final Logger LOG = LoggerFactory.getLogger(dw_get_bribery.class);
   public void dw_get_bribery_run(Conn conn,Connection hconn,Configuration conf)throws Exception{
// public static void main(String[] args) throws Exception{

   String[] familyColumn= new String[]{"USERS","BRIBERY"};
   SparkHBase sparkH=new SparkHBase();    
   sparkH.createTable(conf,"DW_GET_BRIBERY", familyColumn);

   Table table = hconn.getTable(TableName.valueOf("DW_GET_BRIBERY"));
   String maxdate= conn.getMaxDate(conf,hconn,table,"BRIBERY","RECEIVE_TIME");
//    maxdate = "2016-10-30 00:00:00";
      Date date  = new Date();
      Timestamp maxdt ;
//    = new Timestamp(date.getTime()) ;

      if (maxdate.length() < 10){
         maxdate = maxdate + "000000";
         maxdt = conn.GetStampByTime1(maxdate);
      }else {
         maxdate = maxdate.substring(0,10) + " 00:00:00";
         maxdt = conn.GetStampByTime(maxdate);
      }


   String tableBribery = "t_user_bribery_money";
   String tableUsers = "t_users";
      Dataset DFBribery = conn.getDataFrame(tableBribery);
      Dataset DFUsers= conn.getDataFrame(tableUsers);
   
   
   DFBribery.createOrReplaceTempView("t_user_bribery_money");
   DFUsers.createOrReplaceTempView("t_users");

   Dataset filteredDF = conn.spark.sql("SELECT tubm.user_id,u.name,\tu.reality_name,  tubm.receive_type, tubm.type,\n" +
         "  tubm.scope,tubm.invest_start, tubm.invest_end, \n" +
         " DATE_FORMAT(tubm.receive_time,'yyyy-MM-dd HH:mm:ss'),DATE_FORMAT(tubm.end_time,'yyyy-MM-dd HH:mm:ss'),\tsum(tubm.amount) amount\tFROM t_user_bribery_money tubm JOIN t_users u ON tubm.user_id = u.id WHERE\ttubm.receive_time >= '"+maxdt+"'GROUP BY tubm.user_id,u.name,\tu.reality_name,  tubm.receive_type, tubm.type,\n" +
         "  tubm.scope,\n" +
         "  tubm.invest_start,\n" +
         "  tubm.invest_end,tubm.receive_time,\ttubm.end_time ORDER BY\ttubm.receive_time\n");


   try {  
      
      //List<Row> userLogonRows = filteredDF.groupBy(filteredDF.col("type_id"),filteredDF.col("time").substr(0, 10).as("dt")).count().alias("cnt").collectAsList();  
      List<Row> userGetBriberyRows = filteredDF.collectAsList();

//    filteredDF.collect().foreach { userGetLoginRow => {
       for (Row userGetBriberyRow : userGetBriberyRows) {
        //  LOGGER.info("user_id="+userGetBriberyRow.getLong(0)+" name="+userGetBriberyRow.getString(1)+" reality_name="+userGetBriberyRow.getString(2));       
         
         String user_id= String.valueOf(userGetBriberyRow.getLong(0));
         String name= String.valueOf(userGetBriberyRow.getString(1));
         String reality_name = String.valueOf(userGetBriberyRow.getString(2));
         

          String receive_type = String.valueOf(userGetBriberyRow.getInt(3));

           String type = String.valueOf(userGetBriberyRow.getInt(4));
          String scope = String.valueOf(userGetBriberyRow.getInt(5));
          String invest_start  = String.valueOf(userGetBriberyRow.getInt(6));
          String invest_end  = String.valueOf(userGetBriberyRow.getInt(7));

          String receive_time  = userGetBriberyRow.getString(8);
          Timestamp ts = conn.GetStampByTime(receive_time);
          String end_time  = userGetBriberyRow.getString(9);

          String amount = String.valueOf(userGetBriberyRow.getDecimal(10));

         
          //String rowKey = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMdd")).toString()+DateFormatUtils.format(ts, "HHmmss")+user_id;
          String Rkey = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMddHHmmss")).toString();
          String id = String.format("%010d", Integer.valueOf(user_id));
          String rowKey  = Rkey + id;
         //LOGGER.info("rowKey= "+rowKey);
          SparkHBase.addRow(table, rowKey, "USERS", "USER_ID", user_id);
         SparkHBase.addRow(table, rowKey, "USERS", "NAME", name);
         SparkHBase.addRow(table, rowKey, "USERS", "REALITY_NAME", reality_name);

         SparkHBase.addRow(table, rowKey, "BRIBERY", "TYPE", type);
         SparkHBase.addRow(table, rowKey, "BRIBERY", "SCOPE", scope);
         SparkHBase.addRow(table, rowKey, "BRIBERY", "INVEST_START", invest_start);
         SparkHBase.addRow(table, rowKey, "BRIBERY", "INVEST_END", invest_end);

         SparkHBase.addRow(table, rowKey, "BRIBERY", "AMOUNT", amount);
         SparkHBase.addRow(table, rowKey, "BRIBERY", "RECEIVE_TYPE", receive_type);
         SparkHBase.addRow(table, rowKey, "BRIBERY", "RECEIVE_TIME", receive_time);
         SparkHBase.addRow(table, rowKey, "BRIBERY", "END_TIME", end_time);
     }

   } catch (Exception e) {
      if(e.getClass().equals(MasterNotRunningException.class)){
         LOG.error("MasterNotRunningException:" + e.toString());
      }
      if(e.getClass().equals(ZooKeeperConnectionException.class)){
         LOG.error("ZooKeeperConnectionException:" + e.toString());
      }
      if(e.getClass().equals(IOException.class)){
         LOG.error("IOException:" + e.toString());
      }
      e.printStackTrace();
   }finally{
//    if(null!=table){
         table.close();
//    }
   }
         
   }

   }