* 新手标已还款追踪数据。dw_yhk_zz

最新推荐文章于 2021-12-12 11:27:44 发布
最新推荐文章于 2021-12-12 11:27:44 发布
阅读量737 收藏
点赞数
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/tantieshuai/article/details/70313649
版权 
import java.io.IOException
import java.sql.Timestamp
import java.util.Date

import DAO.{ScalaConn, ScalaHbase}
import org.apache.commons.lang3.time.DateFormatUtils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.client._
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.{MasterNotRunningException, TableName, ZooKeeperConnectionException}
import org.apache.spark.sql.DataFrame
import org.slf4j.LoggerFactory

/*
* 新手标已还款追踪数据。
* author：谭志坚
* */


@SerialVersionUID(-1)
object dw_yhk_zz{
  def LOG = LoggerFactory.getLogger(getClass)

  @throws(classOf[Exception])
  def dw_yhk_zz_run( conf: Configuration, hconn: Connection){
    val familyColumn: Array[String] = Array[String]("USERS","INFO")

    ScalaHbase.createTable(conf,hconn, "DW_YHK_ZZ", familyColumn)
    ScalaHbase.createTable(conf,hconn, "DW_YHK_WCZ", familyColumn)
    ScalaHbase.createTable(conf,hconn, "DW_YHK_HZ", familyColumn)
    val spark =  ScalaConn.spark

    val DW_YHK_ZZ: Table = hconn.getTable(TableName.valueOf("DW_YHK_ZZ"))
    val DW_YHK_WCZ: Table = hconn.getTable(TableName.valueOf("DW_YHK_WCZ"))
    val DW_YHK_HZ: Table = hconn.getTable(TableName.valueOf("DW_YHK_HZ"))

    var maxdate: String = ScalaConn.getMaxDate(conf,hconn, DW_YHK_ZZ, "USERS", "DT")
//    var DW_YHK_WCZ_maxdate: String = ScalaConn.getMaxDate(conf,hconn, DW_YHK_WCZ, "USERS", "DT")
//    maxdate = "2017-01-01 00:00:00"
//    val date: Date = new Date();
//    var maxdt: Timestamp = new Timestamp(date.getTime) ;

   var maxdt:String= String.valueOf(ScalaConn.getYesterday(-7,"yyyy-MM-dd HH:mm:ss"))
    maxdt = maxdt.substring(0, 10) + " 00:00:00"

//    if (maxdate<maxdt){
//       maxdt = maxdate
//    }


//    if (maxdate.length < 10){
//      maxdate = maxdate + "000000"
//      maxdt = ScalaConn.GetStampByTime1(maxdate)
//    }else {
//      maxdate = maxdate.substring(0, 10) + " 00:00:00"
//      maxdt = ScalaConn.GetStampByTime(maxdate)
//    }




    val table_planing_bill:String ="t_planing_bill_invests"
    val table_fxgo_bill:String ="t_fxgo_bill_invests"
    val table_bill:String ="t_bill_invests"
    val table_bill_huoqibao:String ="t_huoqibao_user_trade_record"

    val tableInvests: String = "t_invests"
    val tableBids: String = "t_bids"
    val tableBidIn:String="t_bid_increase_interest"
    val tableUsers: String = "t_users"
    val tablFxgo: String = "t_fxgo_invests"
    val tablTtz:String ="t_huoqibao_user_invest"
    val tablePinvest: String = "t_planing_invests"
    val tablePlanings: String = "t_planing"
    val tableDetails: String = "t_user_details"



    val DFplaning_bill: DataFrame = ScalaConn.getP2PDataFrame(table_planing_bill)
    val DFfxgo_bill: DataFrame = ScalaConn.getP2PDataFrame(table_fxgo_bill)
    val DFbill: DataFrame = ScalaConn.getP2PDataFrame(table_bill)
    val DFbill_huoqibao: DataFrame = ScalaConn.getP2PDataFrame(table_bill_huoqibao)

    val DFInvests: DataFrame = ScalaConn.getP2PDataFrame(tableInvests)
    val DFBids: DataFrame = ScalaConn.getP2PDataFrame(tableBids)
    val DFBidIn: DataFrame = ScalaConn.getP2PDataFrame(tableBidIn)

    val DFUsers: DataFrame = ScalaConn.getP2PDataFrame(tableUsers)
    val DFFxgo: DataFrame = ScalaConn.getP2PDataFrame(tablFxgo)
    val DFTtz:DataFrame = ScalaConn.getP2PDataFrame(tablTtz)
    val DFPinvest: DataFrame = ScalaConn.getP2PDataFrame(tablePinvest)
    val DFPlanings: DataFrame = ScalaConn.getP2PDataFrame(tablePlanings)
    val DFDetails: DataFrame = ScalaConn.getP2PDataFrame(tableDetails)

    DFplaning_bill.createOrReplaceTempView("t_planing_bill_invests")
    DFfxgo_bill.createOrReplaceTempView("t_fxgo_bill_invests")
    DFbill.createOrReplaceTempView("t_bill_invests")
    DFbill_huoqibao.createOrReplaceTempView("t_huoqibao_user_trade_record")

    DFInvests.createOrReplaceTempView("t_invests")
    DFBids.createOrReplaceTempView("t_bids")
    DFBidIn.createOrReplaceTempView("t_bid_increase_interest")
    DFUsers.createOrReplaceTempView("t_users")
    DFFxgo.createOrReplaceTempView("t_fxgo_invests")
    DFTtz.createOrReplaceTempView("t_huoqibao_user_invest")
    DFPinvest.createOrReplaceTempView("t_planing_invests")
    DFPlanings.createOrReplaceTempView("t_planing")
    DFDetails.createOrReplaceTempView("t_user_details")






    val userSql = "SELECT a.user_id,\n\ta. NAME,\n\ta.reality_name,\n\ta.mobile,\n\ta.time,\n\ta.special_invitation_code,\n\ta.title,\n\ta.periods,\n\ta.hk_amt,\n\ta.hk_dt,\n\tc.withdraw,\n\tc.dt,\n\tc.periods,\n\tc.plan_amount,\n\tc.new_amount,\n\tc.xdt_amount,\n\tc.ttz_amount,\n\tc.fxgo_amount\nFROM\n\t(\n\t\tSELECT\n\t\t\ttpi.user_id,\n\t\t\ttp.cycle_days periods,\n\t\t\ttpi.time dt,\n\t\t\tSUM(tpi.amount) plan_amount,\n\t\t\t'0' new_amount,\n\t\t\t'0' xdt_amount,\n\t\t\t'0' ttz_amount,\n\t\t\t'0' fxgo_amount,\t\t\t\n\t\t\t'0' withdraw\n\t\tFROM\n\t\t\tt_planing_invests tpi\n\t\tJOIN t_planing tp ON tpi.planing_id = tp.id\n\t\tWHERE\n\t\t\ttpi.transfer_id = 0\n\t\tAND tpi.time >= '"+maxdt+"'\n\t\tGROUP BY\n\t\t\ttpi.user_id,\n\t\t\ttp.cycle_days,\n\t\t\ttpi.time\n\t\tUNION ALL\n\t\t\tSELECT\n\t\t\t\tts.user_id,\n\t\t\t\t(\n\t\t\t\t\tCASE\n\t\t\t\t\tWHEN tb.period_unit = 0 THEN\n\t\t\t\t\t  tb.period *30\n\t\t\t\t\tWHEN tb.period_unit = 1 THEN\n\t\t\t\t\t\ttb.period\n\t\t\t\t\tWHEN tb.period_unit = - 1 THEN\n\t\t\t\t\t\ttb.period *12 *30\n\t\t\t\t\tEND\n\t\t\t\t) AS periods,\n\t\t\t\tts.time dt,\n\t\t\t\t'0' plan_amount,\n\t\t\t\tsum(\n\n\t\t\t\t\tIF (tnew.bid_id > 0, ts.amount, 0)\n\t\t\t\t) new_amount,\n\t\t\t\tsum(\n\n\t\t\t\t\tIF (tnew.bid_id > 0, 0, ts.amount)\n\t\t\t\t) xdt_amount,\n\t\t\t\t'0' ttz_amount,\n\t\t\t\t'0' fxgo_amount,\t\t\t\n\t\t\t\t'0' withdraw\n\t\t\tFROM\n\t\t\t\tt_invests ts\n\t\t\tJOIN t_bids tb ON ts.bid_id = tb.id\n\t\t\tLEFT JOIN (\n\t\t\t\tSELECT\n\t\t\t\t\tbid_id\n\t\t\t\tFROM\n\t\t\t\t\tt_bid_increase_interest\n\t\t\t\tWHERE\n\t\t\t\t\tis_novice = 1\n\t\t\t) tnew ON ts.bid_id = tnew.bid_id\n\t\t\tWHERE\n\t\t\t\tts.transfers_id = 0\n\t\t\tAND ts.time >= '"+maxdt+"'\n\t\t\tGROUP BY\n\t\t\t\tts.user_id,\n\t\t\t\ttb.period_unit,\n\t\t\t\ttb.period,\n\t\t\t\tts.time\n\t\t\tUNION ALL\n\t\t\t\tSELECT\n\t\t\t\t\tuser_id,\n\t\t\t\t\t'1' periods,\n\t\t\t\t\tapply_date dt,\n\t\t\t\t\t'0' plan_amount,\n\t\t\t\t\t'0' new_amount,\n\t\t\t\t\t'0' xdt_amount,\n\t\t\t\t\tsum(apply_amount) ttz_amount,\n\t\t\t\t\t'0' fxgo_amount,\t\t\t\t\t\n\t\t\t\t\t'0' withdraw\n\t\t\t\tFROM\n\t\t\t\t\tt_huoqibao_user_invest\n\t\t\t\tWHERE\n\t\t\t\t\tSTATUS = 1\n\t\t\t\tAND apply_date >= '"+maxdt+"'\n\t\t\t\tGROUP BY\n\t\t\t\t\tuser_id,\n\t\t\t\t\tapply_date\n\t\t\t\tUNION ALL\n\t\t\t\t\tSELECT\n\t\t\t\t\t\tts.user_id,\n\t\t\t\t\t\tts.periods * 30 periods,\n\t\t\t\t\t\tts.time dt,\n\t\t\t\t\t\t'0' plan_amount,\n\t\t\t\t\t\t'0' new_amount,\n\t\t\t\t\t\t'0' xdt_amount,\n\t\t\t\t\t\t'0' ttz_amount,\n\t\t\t\t\t\tsum(ts.amount) fxgo_amount,\t\t\t\t\t\t\n\t\t\t\t\t\t'0' withdraw\n\t\t\t\t\tFROM\n\t\t\t\t\t\tt_fxgo_invests ts\n\t\t\t\t\tWHERE\n\t\t\t\t\t\tts. STATUS = 1\n\t\t\t\t\tAND ts.time >= '"+maxdt+"'\n\t\t\t\t\tGROUP BY\n\t\t\t\t\t\tts.user_id,\n\t\t\t\t\t\tts.periods,\n\t\t\t\t\t\tts.time\n\t\t\t\t\tUNION ALL\n\t\t\t\t\t\tSELECT\n\t\t\t\t\t\t\tuser_id,\n\t\t\t\t\t\t\t'0' periods,\n\t\t\t\t\t\t\ttime dt,\n\t\t\t\t\t\t\t'0' plan_amount,\n\t\t\t\t\t\t\t'0' new_amount,\n\t\t\t\t\t\t\t'0' xdt_amount,\n\t\t\t\t\t\t\t'0' ttz_amount,\n\t\t\t\t\t\t\t'0' fxgo_amount,\t\t\t\t\t\t\n\t\t\t\t\t\t\tsum(\n\n\t\t\t\t\t\t\t\tIF (\n\t\t\t\t\t\t\t\t\toperation = 304\n\t\t\t\t\t\t\t\t\tOR operation = 379,\n\t\t\t\t\t\t\t\t\tIFNULL(amount, 0),\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\t\t\t) withdraw\n\t\t\t\t\t\tFROM\n\t\t\t\t\t\t\tt_user_details\n\t\t\t\t\t\tWHERE\n\t\t\t\t\t\t\toperation IN (304, 379)\n\t\t\t\t\t\tAND time >= '"+maxdt+"'\n\t\t\t\t\t\tGROUP BY\n\t\t\t\t\t\t\tuser_id,\n\t\t\t\t\t\t\ttime\n\t) c\nRIGHT  JOIN (\n\tSELECT\n\t\ttb.user_id,\n\t\tu. NAME,\n\t\tu.reality_name,\n\t\tu.mobile,\n\t\tu.time,\n\t\tu.special_invitation_code,\n\t\ttb.bid_id,\n\t\ttb.title,\n\t\tIFNULL(bid.period, 0) AS periods,\n\t\ttb.real_receive_time hk_dt,\t\t\n\t\tsum(\n\t\t\ttb.real_receive_corpus + tb.real_receive_interest\n\t\t) AS hk_amt\n\tFROM\t\t\n\t\t\t\tt_bill_invests tb\n\t\tJOIN (\n\t\t\t\tSELECT\n\t\t\t\t\tbid_id\n\t\t\t\tFROM\n\t\t\t\t\tt_bid_increase_interest\n\t\t\t\tWHERE\n\t\t\t\t\tis_novice = 1\n\t\t\t) tnew ON tb.bid_id = tnew.bid_id\t\t\t\n\tJOIN t_users u ON tb.user_id = u.id\n  JOIN t_bids bid ON bid.id = tb.bid_id\n\tWHERE tb.real_receive_time >= '"+maxdt+"'\t\n  and tb.user_id not in (63,16721,130845,9944,75,9677,163,9607,16648,16668)\n  and tb.real_receive_corpus+tb.real_receive_interest >0\n\tGROUP BY\n\t\ttb.user_id,\n\t\tu. NAME,\n\t\tu.reality_name,\n\t\tu.mobile,\n\t\tu.time,\n\t\tu.special_invitation_code,\n\t\ttb.bid_id,\n\t\ttb.title,\n\t\tbid.period,\n\t\ttb.real_receive_time\t\n) a ON a.user_id = c.user_id\nWHERE a.hk_dt <= c.dt"

    var userDF: DataFrame = spark.sql(userSql)
    try {
      userDF.collect().foreach { userRow => {
        val user_id: String = String.valueOf(userRow.getLong(0))
        val name: String = String.valueOf(userRow.getString(1))
        val reality_name: String = String.valueOf(userRow.getString(2))
        val mobile: String = String.valueOf(userRow.getString(3))
        val reg_time: String = String.valueOf(userRow.getTimestamp(4))
        val special_invitation_code: String = String.valueOf(userRow.getString(5))
        val title: String =  String.valueOf(userRow.getString(6))
        val bid_periods: String =  String.valueOf(userRow.getInt(7))
        val hk_amt: String =  String.valueOf(userRow.getDecimal(8))
        val hk_dt: String =  String.valueOf(userRow.getTimestamp(9))
        val ts2:Timestamp  =ScalaConn.GetStampByTime(hk_dt)
        val withdraw: String =  String.valueOf(userRow.getString(10))

        var dt: String =""
        var Rkey2: String=""
        val value12 = userRow.isNullAt(11)
        if (value12 != true) {
          dt = String.valueOf(userRow.getTimestamp(11))
          val ts:Timestamp  =ScalaConn.GetStampByTime(dt)
          Rkey2 = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMddHHmmss")).toString
        }
//        val dt: String =  String.valueOf(userRow.getTimestamp(14))




        var periods: String = ""
        val value13 = userRow.isNullAt(12)
        if (value13 != true) {
          periods = String.valueOf(userRow.getString(12))
        }
//        val periods: String =  String.valueOf(userRow.getDouble(14))
        val plan_amount: String =  String.valueOf(userRow.getString(13))
        val new_amount: String =  String.valueOf(userRow.getString(14))
        val xdt_amount: String =  String.valueOf(userRow.getString(15))
        val ttz_amount: String =  String.valueOf(userRow.getString(16))
        val fxgo_amount: String =  String.valueOf(userRow.getString(17))

        //        val Rkey: String = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMdd HH:mm:ssHH")).toString

        val Rkey1: String = new StringBuffer(DateFormatUtils.format(ts2, "yyyyMMddHHmmss")).toString
        var rowKey: String=Rkey1
        val id:String = String.format(String.format("%010d",Integer.valueOf(user_id)))
        if (Rkey2 != "") {
           rowKey= rowKey + Rkey2 + id
        }else{
           rowKey= rowKey + rowKey + id
        }

        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "USER_ID", user_id)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "NAME", name)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "REALITY_NAME", reality_name)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "MOBILE", mobile)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "REG_TIME", reg_time)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "SPECIAL_INVITATION_CODE", special_invitation_code)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "DT", dt)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "TITLE", title)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "BIDPERIODS", bid_periods)

        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "HKAMT", hk_amt)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "HKDT", hk_dt)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "PERIODS", periods)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "WITHDRAW", withdraw)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "FJH", plan_amount)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "NEW", new_amount)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "XDT", xdt_amount)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "TTZ", ttz_amount)
        ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "FXG", fxgo_amount)
      }
      }
    }
    catch {
      case e: Exception => {
        if (e.getClass == classOf[MasterNotRunningException]) {
          LOG.error("MasterNotRunningException:"+e.toString)
        }
        if (e.getClass == classOf[ZooKeeperConnectionException]) {
          LOG.error("ZooKeeperConnectionException:"+e.toString)
        }
        if (e.getClass == classOf[IOException]) {
          LOG.error("IOException:"+e.toString)
        }
        LOG.error("error xception:"+ e.printStackTrace)

      }
    } finally {
      userDF = null
      DW_YHK_ZZ.close
    }

    conf.set(TableInputFormat.INPUT_TABLE, "DW_YHK_ZZ")
    import spark.implicits._
    var hbaseRDD = spark.sparkContext.newAPIHadoopRDD(conf, classOf[TableInputFormat], classOf[ImmutableBytesWritable], classOf[Result])
    var HbaseDF = hbaseRDD.map(r=>(
      Bytes.toString(r._2.getRow),
      Bytes.toString(r._2.getValue(Bytes.toBytes("USERS"),Bytes.toBytes("USER_ID"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("USERS"),Bytes.toBytes("REG_TIME"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("USERS"),Bytes.toBytes("DT"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("HKAMT"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("HKDT"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("PERIODS"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("WITHDRAW"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("FJH"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("NEW"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("XDT"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("TTZ"))),
      Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("FXG")))
      )).toDF("ROWKEY","USER_ID","REG_TIME","DT","HKAMT","HKDT","PERIODS","WITHDRAW","FJH","NEW","XDT","TTZ","FXG")
    HbaseDF.createOrReplaceTempView("DW_YHK_ZZ")

    val DW_YHK_WCZ_userSql ="select wcz.user_id,sum(wcz.HKAMT) HKAMT,sum(wcz.invtamt) invtamt,sum(wcz.WITHDRAW) WITHDRAW, sum(wcz.wcz_amt) wcz_amt from (\nSELECT w3.user_id,sum(w3.HKAMT) HKAMT,0 invtamt,0 WITHDRAW,sum(w3.HKAMT) wcz_amt from (\nselect t1.USER_ID,t1.HKAMT from (\nselect DISTINCT dyz2.USER_ID,dyz2.HKAMT,dyz2.HKDT, dyz2.DT,dyz2.PERIODS,dyz2.WITHDRAW,dyz2.FJH,dyz2.NEW,dyz2.XDT,dyz2.TTZ,dyz2.FXG from DW_YHK_ZZ dyz2  where  dyz2.HKDT >='"+maxdt+"'\n)t1 WHERE T1.DT IS NULL\n) w3 group by w3.user_id\nunion all\nSELECT w1.user_id user_id,w1.HKAMT HKAMT,w2.invtamt invtamt,w2.WITHDRAW WITHDRAW,w1.HKAMT-w2.invtamt-w2.WITHDRAW AS wcz_amt from \n(SELECT t3.user_id,sum(t3.HKAMT) HKAMT from (\nselect distinct dyz3.USER_ID,dyz3.REG_TIME,dyz3.HKAMT,dyz3.HKDT from DW_YHK_ZZ dyz3  \nwhere  dyz3.HKDT >='"+maxdt+"'  \n)t3 GROUP BY t3.user_id) w1 join \n(select t2.USER_ID,sum(t2.WITHDRAW) WITHDRAW,sum(t2.FJH+t2.NEW+t2.XDT+t2.TTZ+t2.FXG) invtamt from (\nselect distinct dyz2.USER_ID,dyz2.DT,dyz2.PERIODS,dyz2.WITHDRAW,dyz2.FJH,dyz2.NEW,dyz2.XDT,dyz2.TTZ,dyz2.FXG from DW_YHK_ZZ dyz2  \nwhere  dyz2.HKDT >='"+maxdt+"' and  dyz2.periods>=0 \n)t2 group by t2.USER_ID) w2 on w1.user_id = w2.USER_ID WHERE w1.HKAMT>(w2.invtamt+w2.WITHDRAW)\n) wcz group by  wcz.user_id"
    var DW_YHK_WCZDF: DataFrame = spark.sql(DW_YHK_WCZ_userSql)
    try {
      DW_YHK_WCZDF.collect().foreach { userRow => {
        val user_id: String = String.valueOf(userRow.getString(0))
        val HKAMT: String = String.valueOf(userRow.getDouble(1))
        val invtamt: String = String.valueOf(userRow.getDouble(2))
        val WITHDRAW: String = String.valueOf(userRow.getDouble(3))
        val wcz_amt: String = String.valueOf(userRow.getDouble(4))


//        ("ROWKEY","USER_ID","REG_TIME","DT","RECHARGE","HKAMT","HKDT","PERIODS","WITHDRAW","FJH","NEW","XDT","TTZ","FXG")
        //          val ts: Timestamp = ScalaConn.GetStampByTime(ymd)
      val date: Date = new Date();
      var ts: Timestamp = new Timestamp(date.getTime) ;
        ts = ScalaConn.GetStampByTime(maxdt)
//        val rowKey: String = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMdd")).toString + DateFormatUtils.format(ts, "HHmmss")
        val Rkey1: String = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMdd")).toString
        val id:String = String.format("%010d",Integer.valueOf(user_id))

        val rowKey = Rkey1 + id
        //        val rowKey: String =  ScalaConn.GetStampByTime1(ymd).toString

        ScalaHbase.addRow(DW_YHK_WCZ, rowKey, "USERS", "USER_ID", user_id)
        ScalaHbase.addRow(DW_YHK_WCZ, rowKey, "USERS", "HKAMT", HKAMT)
        ScalaHbase.addRow(DW_YHK_WCZ, rowKey, "USERS", "INVTAMT", invtamt)
        ScalaHbase.addRow(DW_YHK_WCZ, rowKey, "USERS", "WITHDRAW", WITHDRAW)
        ScalaHbase.addRow(DW_YHK_WCZ, rowKey, "USERS", "WCZ_AMT", wcz_amt)
      }
      }
    }
    catch {
      case e: Exception => {
        if (e.getClass == classOf[MasterNotRunningException]) {
          LOG.error("MasterNotRunningException:" + e.toString)
        }
        if (e.getClass == classOf[ZooKeeperConnectionException]) {
          LOG.error("ZooKeeperConnectionException:" + e.toString)
        }
        if (e.getClass == classOf[IOException]) {
          LOG.error("IOException:" + e.toString)
        }
        e.printStackTrace
      }
    } finally {
      DW_YHK_WCZDF = null
      if (null != DW_YHK_WCZ) {
        DW_YHK_WCZ.close
      }
    }

    val DW_YHK_HZ_userSql ="SELECT w1.user_id,w1.HKAMT,w2.PERIODS,w2.WITHDRAW,w2.FJH,w2.NEW,w2.XDT,w2.TTZ,w2.FXG from \n(\nSELECT t3.user_id,sum(t3.HKAMT) HKAMT from (\nselect distinct dyz3.USER_ID,dyz3.REG_TIME,dyz3.HKAMT,dyz3.HKDT from DW_YHK_ZZ dyz3  \nwhere  dyz3.HKDT >='"+maxdt+"'  \n)t3 GROUP BY t3.user_id) w1 \nLEFT JOIN\n(\nselect t2.USER_ID,t2.PERIODS,sum(t2.WITHDRAW) WITHDRAW,sum(t2.FJH) FJH,SUM(t2.NEW) NEW,SUM(t2.XDT) XDT,SUM(t2.TTZ) TTZ,SUM(t2.FXG) FXG from \n(select distinct dyz2.USER_ID,dyz2.DT,dyz2.PERIODS,dyz2.WITHDRAW,dyz2.FJH,dyz2.NEW,dyz2.XDT,dyz2.TTZ,dyz2.FXG from DW_YHK_ZZ dyz2  \nwhere  dyz2.HKDT >='"+maxdt+"' and  dyz2.periods>=0  \n)t2 group by t2.USER_ID,t2.PERIODS\n) w2 on w1.user_id = w2.USER_ID"

    var DW_YHK_HZDF: DataFrame = spark.sql(DW_YHK_HZ_userSql)
    try {
      DW_YHK_HZDF.collect().foreach { userRow => {
        val user_id: String = String.valueOf(userRow.getString(0))
        val HKAMT: String = String.valueOf(userRow.getDouble(1))
        val PERIODS: String = String.valueOf(userRow.getString(2))

        var WITHDRAW: String = "0"
        val value3 = userRow.isNullAt(3)
        if (value3 != true) {
          WITHDRAW = String.valueOf(userRow.getDouble(3))
        }

//        val WITHDRAW: String = String.valueOf(userRow.getDouble(3))
        val FJH: String = String.valueOf(userRow.getDouble(4))
        val NEW: String = String.valueOf(userRow.getDouble(5))
        val XDT: String = String.valueOf(userRow.getDouble(6))
        val TTZ: String = String.valueOf(userRow.getDouble(7))
        val FXG: String = String.valueOf(userRow.getDouble(8))

        val date: Date = new Date();
        var ts: Timestamp = new Timestamp(date.getTime) ;
        ts = ScalaConn.GetStampByTime(maxdt)
        //        val rowKey: String = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMdd")).toString + DateFormatUtils.format(ts, "HHmmss")
        val Rkey1: String = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMdd")).toString
        val id:String = String.format("%010d",Integer.valueOf(user_id))
        val periodsid:String = String.format("%04d",Integer.valueOf(PERIODS))

        val rowKey = Rkey1 + id+periodsid

        ScalaHbase.addRow(DW_YHK_HZ, rowKey, "USERS", "USER_ID", user_id)
        ScalaHbase.addRow(DW_YHK_HZ, rowKey, "USERS", "HKAMT", HKAMT)
        ScalaHbase.addRow(DW_YHK_HZ, rowKey, "USERS", "PERIODS", PERIODS)
        ScalaHbase.addRow(DW_YHK_HZ, rowKey, "USERS", "WITHDRAW", WITHDRAW)
        ScalaHbase.addRow(DW_YHK_HZ, rowKey, "USERS", "FJH", FJH)
        ScalaHbase.addRow(DW_YHK_HZ, rowKey, "USERS", "NEW", NEW)
        ScalaHbase.addRow(DW_YHK_HZ, rowKey, "USERS", "XDT", XDT)
        ScalaHbase.addRow(DW_YHK_HZ, rowKey, "USERS", "TTZ", TTZ)
        ScalaHbase.addRow(DW_YHK_HZ, rowKey, "USERS", "FXG", FXG)
      }
      }
    }
    catch {
      case e: Exception => {
        if (e.getClass == classOf[MasterNotRunningException]) {
          LOG.error("MasterNotRunningException:" + e.toString)
        }
        if (e.getClass == classOf[ZooKeeperConnectionException]) {
          LOG.error("ZooKeeperConnectionException:" + e.toString)
        }
        if (e.getClass == classOf[IOException]) {
          LOG.error("IOException:" + e.toString)
        }
        e.printStackTrace
      }
    } finally {
      DW_YHK_HZDF = null
      if (null != DW_YHK_WCZ) {
        DW_YHK_HZ.close
      }
    }
 }
}
tantieshuai
关注
数据结构内容总结(顺序表)yhk
2302_79795375的博客
03-30 766
笔者写本文有三个作用，1.笔者由于刚学完数据结构的内容想要对所学内容进行总结；2.笔者也想写出总结内容为读者提供复习的机会；3.笔者也希望能得到各位看官的笔者所总结内容的评价。笔者会用c语言实现代码提示：以下是本篇文章正文内容，下面案例可供参考提示：这里对文章进行总结：例如：以上就是今天要讲的内容，本文仅仅简单介绍了pandas的使用，而pandas提供了大量能使我们快速便捷地处理数据的函数和方法。
【数据分析报告】携程客户分析与流失预测
热门推荐
changreal的博客
08-11 7万+
目录一、项目背景与目的二、探索性分析2.1 数据指标预览2.2 数据概况2.3 数据分布2.3.1 数据分布总览2.3.2 预定日期和入住日期2.3.3 访问时间段2.3.4 客户价值2.3.5 消费能力指数2.3.6 价格敏感指数分布2.3.6 入住酒店平均价格2.3.7 酒店星级偏好2.3.8 订单取消率2.3.9 用户年订单数分布2.3.10 新老客户流失率三、数据预处理3.1 去除不需要的字段与重复字段3.2 数据类型转换3.3 异常值处理3.3.1负数处理3.3.2 极值处理3.4 缺失值处理3.
IDEA出现Unable to resolve column 。。。。问题的完美解决方法
weixin_39940206的博客
07-22 2万+
出现问题是Unable to resolve column。。。 解决方法： idea中file——>setting——>languages&Frameworks——>SQL Resolution Scopes——>Project mapping选择自己项目的库表——>点击加号加上自己的项目地址——>Resolution Scope下一样选择自己的库表——>Apply——>OK就完美解决了。 1. 2. 3. 4. 5. 我也不知道为什么这
java对称加密_php对称解密,php与java对称加密解密
weixin_39929595的博客
04-13 208
```php```javapackage com.gdin.yhk.common.util;import java.io.UnsupportedEncodingException;import java.security.NoSuchAlgorithmException;import java.security.SecureRandom;import java.security.Security;...
1万条数据大概占多大空间_快递信息泄露背后：40元可买超4G简历数据
weixin_39894104的博客
11-18 220
贝壳财经原创出品记者 李大伟编辑 李薇佳被圆通快递内鬼泄露的四十万公民个人信息只是庞大信息贩卖江湖的冰山一角。11月16日，新京报贝壳财经独家报道了圆通多位“内鬼”有偿租借员工账号，40万条公民个人信息被泄露一事，引起大众对于个人隐私安全问题的重视。贝壳财经报道，在由邯郸市公安局反诈中心联合邯郸市永年区公安局成立的专案组近期侦办的一起部督案件中，不法分子与圆通快递多位“内鬼”勾结，通过有...
dm_zj_m月度总结数据
tantieshuai的博客
04-21 559
import java.io.IOException import java.sql.Timestamp import java.text.SimpleDateFormat import java.util.Calendar import DAO.{ScalaConn, ScalaHbase} import org.apache.commons.lang3.time.DateFormatUtil
1万条数据大概占多大空间_圆通内鬼泄露40万条个人信息背后：不仅快递，你的简历也被卖了-科技频道...
weixin_39939904的博客
11-19 341
被圆通快递内鬼泄露的四十万条公民个人信息只是庞大信息贩卖江湖的冰山一角。11月16日，新京报贝壳财经独家报道了圆通多位“内鬼”有偿租借员工账号，40万条公民个人信息被泄露一事，引起大众对于个人隐私安全问题的重视。贝壳财经报道，在由邯郸市公安局反诈中心联合邯郸市永年区公安局成立的专案组近期侦办的一起部督案件中，不法分子与圆通快递多位“内鬼”勾结，通过有偿租用圆通员工系统账号盗取公民个人信息，再层层倒...
清华操作系统ucore_lab1
JustinAustin的博客
12-12 1748
操作系统实验报告 练习一 一、操作系统镜像文件ucore.img是如何一步一步生成的？(需要比较详细地解释Makefile中每一条相关命令和命令参数的含义，以及说明命令导致的结果) 首先我们在Makefile文档中找到ucore.img文件生成的代码段： [外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-DHSL6Ce1-1639279618160)(E:/Typore%E5%9B%BE%E7%89%87/image-20211116160117324.png)] 我们对该代码进
mac ssh安装mysql_Mac安装MySQL-python报错解决
weixin_32688511的博客
01-19 154
Mac安装MySQL-pyhton报错今天在Mac上安装MySQL-python报错，搜遍网络都说什么mysql config路径问题，但是都不行。解决方案一开始遇到的问题是：Complete output from command python setup.py egg_info:sh: mysql_config: command not foundTraceback (most recent ...
YHK-M290_BK3632_V1_0.PCB
07-25
YHK-M290_BK3632_V1_0.PCB
YHK7000A开关柜智能操控装置.pdf
10-13
《YHK7000A开关柜智能操控装置》是安徽亚辉电气自动化有限公司推出的一款高效、智能化的电力设备，其设计目标在于提升开关柜的操作安全性和管理效率。这款装置集成了多种功能，旨在实现对开关柜的全方位监控与控制。...
科士达 UPS 通讯协议 GP800 YDC YHK 系列
06-05
《科士达 UPS 通讯协议 GP800 YDC YHK 系列详解》 在IT行业中，不间断电源（Uninterruptible Power Supply, UPS）扮演着至关重要的角色，尤其是在数据中心和关键业务系统中，它保障了电力供应的稳定性和可靠性。...
VP2468a_UG_SCH
06-20
【VP2468a液晶显示器用户指南】是ViewSonic公司的一款专业级显示器的使用手册，型号为VS16475。该显示器旨在提供卓越的视觉体验，适合需要高质量显示效果的用户。在开始使用VP2468a之前，用户必须遵循一系列的安全...
【重磅，更新！】2002-2021年中国31省份经济韧性测度三级指标数据合集（各省、市、企业等）
11-12
1、资源内容地址：https://blog.csdn.net/abc6838/article/details/143720369 2、数据特点：今年全新，手工精心整理，放心引用，数据来自权威，且标注《数据来源》，相对于其他人的控制变量数据准确很多,适合写论文做实证用 ，不会出现数据造假问题 3、适用对象：大学生，本科生，研究生小白可用，容易上手！！！ 4、课程引用: 经济学，地理学，城市规划与城市研究，公共政策与管理，社会学，商业与管理
CPPC++_更好的Windows字体渲染.zip
11-12
CPPC++_更好的Windows字体渲染
10018.doc
11-12
10018
在Windows capa中轻松创建虚拟显示.zip
11-12
cppc++
二环北路东段欣心家园小区商业B段(中石油加油站东邻).m4a
最新发布
11-12
二环北路东段欣心家园小区商业B段(中石油加油站东邻).m4a
CPPC++_自主无人机的开源软件.zip
11-12
cppc++
sqlserver数据库中group_concat函数
08-25
SELECT name, STUFF((SELECT ',' + bankCard FROM uf_yhk t2 WHERE t1.name = t2.name FOR XML PATH('')), 1, 1, '') AS bankCards FROM uf_yhk t1 GROUP BY name ``` 在这个查询中，我们使用了STUFF函数和FOR XML ...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值