sparkToHbase

最新推荐文章于 2022-03-24 21:40:45 发布

qzWsong

最新推荐文章于 2022-03-24 21:40:45 发布

阅读量155

点赞数

分类专栏： spark

本文链接：https://blog.csdn.net/java_creatMylief/article/details/113280092

版权

spark 专栏收录该内容

27 篇文章 0 订阅

订阅专栏

Cid,关联 order_goods_type 获取订单商品名称，然后导入hbase

{"oid":"o123", "cid": 1, "money": 600.0, "longitude":116.397128,"latitude":39.916527}
oid":"o112", "cid": 3, "money": 200.0, "longitude":118.396128,"latitude":35.916527}
{"oid":"o124", "cid": 2, "money": 200.0, "longitude":117.397128,"latitude":38.916527}
{"oid":"o125", "cid": 3, "money": 100.0, "longitude":118.397128,"latitude":35.916527}
{"oid":"o127", "cid": 1, "money": 100.0, "longitude":116.395128,"latitude":39.916527}
{"oid":"o128", "cid": 2, "money": 200.0, "longitude":117.396128,"latitude":38.916527}
{"oid":"o129", "cid": 3, "money": 300.0, "longitude":115.398128,"latitude":35.916527}
{"oid":"o130", "cid": 2, "money": 100.0, "longitude":116.397128,"latitude":39.916527}
{"oid":"o131", "cid": 1, "money": 100.0, "longitude":117.394128,"latitude":38.916527}
{"oid":"o132", "cid": 3, "money": 200.0, "longitude":118.396128,"latitude":35.916527}

package com.ws.orderCount


case class Order(cid:String,oid:String,money:Double,var cname:String,longitude:Double,latitude:Double) {
  override def toString: String = {
    s"cid:$cid,oid:$oid,money:$money,cname:$cname,longitude:$longitude,latitude:$latitude"
  }
}

package com.ws.orderCount

import java.sql.{DriverManager, ResultSet}
import java.util

import com.google.gson.Gson
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.client.{Connection, ConnectionFactory, Put, Table}
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.{HBaseConfiguration, TableName}
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import org.slf4j.{Logger, LoggerFactory}

object OrderCountToHbase {
  private val logger: Logger = LoggerFactory.getLogger(OrderCountToHbase.getClass)

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("OrderCountToHbase").setMaster("local[*]")
    val sc = new SparkContext(conf)
    val files = sc.textFile("data/order.log")
    val orders: RDD[Order] = files.mapPartitions(x => {
      val gson = new Gson()
      val res: Iterator[Order] = x.map(x => {
        try {
          gson.fromJson(x, classOf[Order])
        } catch {
          case e: Exception =>
            e.printStackTrace()
            logger.error("json err【" + x + "】")
            null
        }
      })
      res
    })
    val orders_f = orders.filter(_ != null)
    val ordersplus: RDD[Order] = orders_f.mapPartitions(x => {
      val conn = DriverManager.getConnection("jdbc:mysql://dream3:3306/bigdata?charactedEncoding=utf-8", "root", "root")
      val st = conn.prepareStatement("select goodsname from order_goods_type where id=?")
      x.map(y => {
        try {
          st.setInt(1, Integer.parseInt(y.cid))
          val set: ResultSet = st.executeQuery()
          while (set.next()) {
            y.cname = set.getString("goodsname")
          }
          if (!x.hasNext) {
            if(set!=null){
              set.close()
            }
            if (st != null) {
              st.close()
            }
            if (conn != null) {
              conn.close()
            }
          }
          y
        } catch {
          case e: Exception =>
            e.printStackTrace()
            null
        }
      })
    })
    println(ordersplus.collect().toBuffer)
    ordersplus.foreachPartition(it => {
      val conf: Configuration = HBaseConfiguration.create()
      conf.set("hbase.zookeeper.quorum", "dream1:2181,dream2:2181,dream3:2181")
      val conn: Connection = ConnectionFactory.createConnection(conf)
      val table: Table = conn.getTable(TableName.valueOf("orders".getBytes()))
      val puts: util.ArrayList[Put] = new util.ArrayList[Put](100)
      try {
        it.foreach(x => {
          val put = new Put(Bytes.toBytes(x.oid))
          put.addColumn("order_info".getBytes(), "cname".getBytes(), x.cname.getBytes())
          put.addColumn("order_info".getBytes(), "money".getBytes(), Bytes.toBytes(x.money))
          put.addColumn("address_info".getBytes(), "longitude".getBytes(), Bytes.toBytes(x.longitude))
          put.addColumn("address_info".getBytes(), "latitude".getBytes(), Bytes.toBytes(x.latitude))
          puts.add(put)
          if (puts.size() == 100) {
            table.put(puts)
            puts.clear()
          }
        })
        table.put(puts)
      } catch {
        case e: Exception =>
          e.printStackTrace()
          logger.error("sssssss")
      } finally {
        if(table!=null){
          table.close()
        }
        if(conn!=null){
          conn.close()
        }
      }
    })
  }
}

结果

hbase(main):009:0> scan 'orders'
ROW                                  COLUMN+CELL                                                                                               
 o123                                column=address_info:latitude, timestamp=1611688596723, value=@C\xF5P\xC1\xB9sT                            
 o123                                column=address_info:longitude, timestamp=1611688596723, value=@]\x19j\x8B\x8F\x14\xDB                     
 o123                                column=order_info:cname, timestamp=1611688596723, value=\xE5\xAE\xB6\xE5\x85\xB7                          
 o123                                column=order_info:money, timestamp=1611688596723, value=@\x82\xC0\x00\x00\x00\x00\x00                     
 o124                                column=address_info:latitude, timestamp=1611688596723, value=@CuP\xC1\xB9sT                               
 o124                                column=address_info:longitude, timestamp=1611688596723, value=@]Yj\x8B\x8F\x14\xDB                        
 o124                                column=order_info:cname, timestamp=1611688596723, value=\xE6\x89\x8B\xE6\x9C\xBA                          
 o124                                column=order_info:money, timestamp=1611688596723, value=@i\x00\x00\x00\x00\x00\x00                        
 o125                                column=address_info:latitude, timestamp=1611688596723, value=@A\xF5P\xC1\xB9sT                            
 o125                                column=address_info:longitude, timestamp=1611688596723, value=@]\x99j\x8B\x8F\x14\xDB                     
 o125                                column=order_info:cname, timestamp=1611688596723, value=\xE6\x9C\x8D\xE8\xA3\x85                          
 o125                                column=order_info:money, timestamp=1611688596723, value=@Y\x00\x00\x00\x00\x00\x00                        
 o127                                column=address_info:latitude, timestamp=1611688596723, value=@C\xF5P\xC1\xB9sT                            
 o127                                column=address_info:longitude, timestamp=1611688596723, value=@]\x19I\xC6\xF3n\xF8                        
 o127                                column=order_info:cname, timestamp=1611688596723, value=\xE5\xAE\xB6\xE5\x85\xB7                          
 o127                                column=order_info:money, timestamp=1611688596723, value=@Y\x00\x00\x00\x00\x00\x00                        
 o128                                column=address_info:latitude, timestamp=1611688596723, value=@CuP\xC1\xB9sT                               
 o128                                column=address_info:longitude, timestamp=1611688596723, value=@]YZ)AA\xEA                                 
 o128                                column=order_info:cname, timestamp=1611688596723, value=\xE6\x89\x8B\xE6\x9C\xBA                          
 o128                                column=order_info:money, timestamp=1611688596723, value=@i\x00\x00\x00\x00\x00\x00                        
 o129                                column=address_info:latitude, timestamp=1611688596721, value=@A\xF5P\xC1\xB9sT                            
 o129                                column=address_info:longitude, timestamp=1611688596721, value=@\x5C\xD9z\xED\xDC\xE7\xCD                  
 o129                                column=order_info:cname, timestamp=1611688596721, value=\xE6\x9C\x8D\xE8\xA3\x85                          
 o129                                column=order_info:money, timestamp=1611688596721, value=@r\xC0\x00\x00\x00\x00\x00                        
 o130                                column=address_info:latitude, timestamp=1611688596721, value=@C\xF5P\xC1\xB9sT                            
 o130                                column=address_info:longitude, timestamp=1611688596721, value=@]\x19j\x8B\x8F\x14\xDB                     
 o130                                column=order_info:cname, timestamp=1611688596721, value=\xE6\x89\x8B\xE6\x9C\xBA                          
 o130                                column=order_info:money, timestamp=1611688596721, value=@Y\x00\x00\x00\x00\x00\x00                        
 o131                                column=address_info:latitude, timestamp=1611688596721, value=@CuP\xC1\xB9sT                               
 o131                                column=address_info:longitude, timestamp=1611688596721, value=@]Y9d\xA5\x9C\x06                           
 o131                                column=order_info:cname, timestamp=1611688596721, value=\xE5\xAE\xB6\xE5\x85\xB7                          
 o131                                column=order_info:money, timestamp=1611688596721, value=@Y\x00\x00\x00\x00\x00\x00                        
 o132                                column=address_info:latitude, timestamp=1611688596721, value=@A\xF5P\xC1\xB9sT                            
 o132                                column=address_info:longitude, timestamp=1611688596721, value=@]\x99Z)AA\xEA                              
 o132                                column=order_info:cname, timestamp=1611688596721, value=\xE6\x9C\x8D\xE8\xA3\x85                          
 o132                                column=order_info:money, timestamp=1611688596721, value=@i\x00\x00\x00\x00\x00\x00