业务是订单成交信息,要求计算出成交总金额,每一类商品的金额,区域成交的金额这三个指标。
数据格式:C 202.102.152.3 家具 婴儿床 2000
SparkStreaming读取Kafka中的数据,使用直连方式,然后实现数据的累加,数据保存到Redis中。
OrderCount.scala
package XXX
import io.netty.handler.codec.string.StringDecoder
import kafka.common.TopicAndPartition
import kafka.message.MessageAndMetadata
import kafka.utils.{ZKGroupTopicDirs, ZkUtils}
import org.I0Itec.zkclient.ZkClient
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka.{HasOffsetRanges, KafkaUtils, OffsetRange}
/**
* Create by ...
*
*/
object OrderCount {
def main(args: Array[String]): Unit = {
//指定组名
val group = "g1"
//创建SparkConf
val conf = new SparkConf().setAppName("KafkaDirectWordCountV2").setMaster("local[4]")
//创建SparkStreaming,并设置间隔时间
val ssc = new StreamingContext(conf,Seconds(5))
//获取广播数据的引用
val broadcastRef = IpUtils.broadcastIpRules(ssc,args(0))
//指定消费者的topic名字
val topic = "orders"
//指定Kafka的broker地址(SparkStreaming的Task直接连到Kafka的分区上,用更加底层的API消费,效率更高)
val brokerList = "L3:9092,L4:9092,L5:9092"
//指定zk的地址,后期更新消费的偏移量时使用(也可以使用Redis,mysql来记录偏移量)
val zkQuroum = "L1:2181,L2:2181,L3:2181"
//创建stream时使用的topic名字集合,SparkStreaming可同时消费多个topic
val topics: Set[String] = Set(topic)
//创建一个ZKGroupTopicDirs对象,其实是指定往zk中写入数据的目录,用于保存偏移量
val topicDirs = new ZKGroupTopicDirs(group,topic)
//获取zookeeper中的路径"g002/offsets/wc"
val zkTopicPath = s"${topicDirs.consumerOffsetDir}"
//准备Kafka的参数
val kafkaParams = Map(
//"key.deserializer" -> classOf[StringDeserializer],
//"value.deserializer" -> classOf[StringDeserializer],
//"deserializer.encoding" -> "GB2312", //配置读取Kafka中数据的编码
"metadata.broker.list" -> brokerList,
"group.id" -> group,
//从头开始读取数据
"auto.offset.reset" -> kafka.api.OffsetRequest.SmallestTimeString
)
//zookeeper的host和IP,创建一个client,用于更新偏移量
//是zookeeper的客户端,可以从zk中读取偏移量数据,并更新偏移量
val zkClient = new ZkClient(zkQuroum)
//查询改路径下是否有节点(默认有子节点是为我们保存不同partition时生成的)
// /g002/offsets/wc/0/10001
// /g002/offsets/wc/1/30001
// /g002/offsets/wc/2/10001
// zkTopicPath -> /g002/offsets/wc/
val children = zkClient.countChildren(zkTopicPath)
var kafkaStream:InputDStream[(String,String)] = null
//如果zookeeper中保存有offset,我们会利用这个offset作为kafkaStream的起始位置
var fromOffsets:Map[TopicAndPartition,Long] = Map()
//如果保存过offset
if (children > 0) {
for (i <- 0 until children) {
// /g002/offsets/wc/0/10001
// /g002/offsets/wc/0
val partitionOffset = zkClient.readData[String](s"$zkTopicPath/$i")
// wc/0
val tp = TopicAndPartition(topic, i)
//将不同 partition 对应的 offset 增加到 fromOffsets 中
// wc/0 -> 10001
fromOffsets += (tp -> partitionOffset.toLong)
}
//Key: kafka的key values: "hello tom hello jerry"
//这个会将 kafka 的消息进行 transform,最终 kafka 的数据都会变成 (kafka的key, message) 这样的 tuple
val messageHandler = (mmd: MessageAndMetadata[String, String]) => (mmd.key(), mmd.message())
//通过KafkaUtils创建直连的DStream(fromOffsets参数的作用是:按照前面计算好了的偏移量继续消费数据)
//[String, String, StringDecoder, StringDecoder, (String, String)]
// key value key的解码方式 value的解码方式
kafkaStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder, (String, String)](ssc, kafkaParams, fromOffsets, messageHandler)
} else {
//如果未保存,根据 kafkaParam 的配置使用最新(largest)或者最旧的(smallest) offset
kafkaStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParams, topics)
}
//偏移量的范围
var offsetRanges = Array[OffsetRange]()
//直连方式只有在KafkaDStream的RDD中才能获取偏移量,那么就不能到调用DStream的Transformation
//所以只能子在kafkaStream调用foreachRDD,获取RDD的偏移量,然后就是对RDD进行操作了
//依次迭代KafkaDStream中的KafkaRDD
//如果使用直连方式累加数据,那么就要在外部的数据库中进行累加(用KeyValue的内存数据库(Nosql),Redis)
//kafkaStream.foreachRDD里面的业务逻辑是在Driver端执行的
kafkaStream.foreachRDD { kafkaRDD =>
//判断当前的kafkaStream中的RDD是否有数据
if (!kafkaRDD.isEmpty()){
//只有KafkaRDD可以强转成HasOffsetRanges,并获取到偏移量
offsetRanges = kafkaRDD.asInstanceOf[HasOffsetRanges].offsetRanges
val lines: RDD[String] = kafkaRDD.map(_._2)
//整理数据
val fields: RDD[Array[String]] = lines.map(_.split(" "))
//1.计算成交总金额
CalculateUtil.calculateIncome(fields)
//2.计算商品分类金额
CalculateUtil.calculateItem(fields)
//3.计算区域成交金额
CalculateUtil.calculateZone(fields,broadcastRef)
//偏移量更新是在Driver端
for (o <- offsetRanges) {
// /g002/offsets/wc/0
val zkPath = s"${topicDirs.consumerOffsetDir}/${o.partition}"
//将该 partition 的 offset 保存到 zookeeper
// /g002/offsets/wc/0/20000
ZkUtils.updatePersistentPath(zkClient, zkPath, o.untilOffset.toString)
}
}
}
ssc.start()
ssc.awaitTermination()
}
}
CalculateUtil.scala
用来计算具体业务的工具类。
package XXX
import cn.edu360.sparkIpTest.TestIp
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
/**
* Create by 。。。
*
*/
object CalculateUtil {
def calculateIncome(fields:RDD[Array[String]]) : Unit = {
//将计算好的数据写到redis中
val priceRDD: RDD[Double] = fields.map(_(4).toDouble)
//reduce是一个Action,会把结果返回到Driver端
//将当前批次的总金额返回了
val sum: Double = priceRDD.reduce(_+_)
//获取一个jedis连接
val conn = jedisConnectionPool.getConnection
//将历史值和当前的值进行累加
//conn.set(Constant.TOTAL_INCOME,sum.toString)
conn.incrByFloat(Constant.TOTAL_INCOME,sum)
//释放连接
conn.close()
}
/**
* 计算分类的成交金额
* @param fields 整理后的数据
*/
def calculateItem(fields:RDD[Array[String]]) : Unit = {
//对fields的map方法是在Driver端执行的
val itemAndPrice: RDD[(String, Double)] = fields.map(arr => {
//取出分类
val item = arr(2)
//取出金额
val price = arr(4).toDouble
(item, price)
})
//按照商品分类进行聚合
val reduced: RDD[(String, Double)] = itemAndPrice.reduceByKey(_+_)
//将当前批次的数据累加到Redis中
//foreachPartition是一个Action
//现在这种方式,jedis是在Driver端创建的
//在Driver端拿jedis连接不好
//val conn = jedisConnectionPool.getConnection()
reduced.foreachPartition(part => {
//获取一个jedis连接
//这个连接是在executor中获取的
//jedisConnectionPool在一个executor进程中只有一个实例(因为jedisConnectionPool是一个object,单例)
val conn = jedisConnectionPool.getConnection
part.foreach(t => {
//一个连接更新多条数据
conn.incrByFloat(t._1,t._2)
})
//将当前分区中的数据更新完再关闭连接
conn.close()
})
}
/**
* 根据IP计算归属地
* @param fields 整理后的数据
* @param broadcastRef 广播数据的引用
*/
def calculateZone(fields:RDD[Array[String]], broadcastRef:Broadcast[Array[(Long, Long, String)]]) : Unit = {
val provinceAndPrice: RDD[(String, Double)] = fields.map(arr => {
val ip = arr(1)
//获取订单金额
val price = arr(4).toDouble
//将数据中的IP转换成二进制
val ipNum = TestIp.ip2Long(ip)
//在executor中获取到广播的全部规则
val allRules: Array[(Long, Long, String)] = broadcastRef.value
//二分法查找
var province = "未知"
val index: Int = TestIp.binarySearch(allRules, ipNum)
if (index != -1) {
province = allRules(index)._3
}
//省份,订单金额
(province, price)
})
//聚合
val reduced: RDD[(String, Double)] = provinceAndPrice.reduceByKey(_+_)
//将结果更新到redis中
reduced.foreachPartition(part => {
//获取jedis连接
val conn = jedisConnectionPool.getConnection
part.foreach(t => {
conn.incrByFloat(t._1,t._2)
})
conn.close()
})
}
}
IpUtils.scala
用来处理广播的IP规则的工具类。
package XXX
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.StreamingContext
/**
* Create by 。。。
*
*/
object IpUtils {
def broadcastIpRules(ssc: StreamingContext, ipRulesPath: String) : Broadcast[Array[(Long, Long, String)]] = {
//先获取sparkContext
val sc = ssc.sparkContext
//将ip.txt读取到HDFS中
val rulesLines: RDD[String] = sc.textFile(ipRulesPath)
//整理ip规则数据
//这里是在Executor中执行的,每个Executor只计算部分的IP规则数据
val ipRulesRDD: RDD[(Long, Long, String)] = rulesLines.map(line => {
val fields = line.split("[|]")
val startNum = fields(2).toLong
val endNum = fields(3).toLong
val province = fields(6)
(startNum, endNum, province)
})
//需要将每个Executor端执行完的数据收集到Driver端
val rulesInDriver: Array[(Long, Long, String)] = ipRulesRDD.collect()
//再将Driver端的完整的数据广播到Executor端
//生成广播数据的引用
val broadcastRef: Broadcast[Array[(Long, Long, String)]] = sc.broadcast(rulesInDriver)
broadcastRef
}
}
jedisConnectionPool.scala
用来获取jedis连接。
package XXX
import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig}
/**
* Create by 。。。
*
*/
object jedisConnectionPool {
val config = new JedisPoolConfig()
//最大连接数
config.setMaxTotal(20)
//最大空闲连接数
config.setMaxIdle(10)
//当调用borrow Object方法时,是否进行有效性检查 -->
config.setTestOnBorrow(true)
//10000代表超时时间(10秒)
val pool = new JedisPool(config,"192.168.67.134",6379,10000,"")
def getConnection:Jedis = {
pool.getResource
}
def main(args: Array[String]): Unit = {
val conn = jedisConnectionPool.getConnection
// conn.set("income","1000")
//
// val r1 = conn.get("tianmao")
// println(r1)
//
// conn.incrBy("tianmao",-20)
//
// val r2 = conn.get("tianmao")
// println(r2)
//
// conn.close()
val r = conn.keys("*")
import scala.collection.JavaConversions._
for (p <- r){
println(p + ":" + conn.get(p))
}
}
}
TestIp.scala
对IP进行处理–将IP转换为十进制,通过二分法查找获取对应的省份
package XXX
import java.sql.{Connection, DriverManager, PreparedStatement}
import scala.io.{BufferedSource, Source}
object TestIp {
//将IP转化为十进制
def ip2Long(ip: String): Long = {
val fragments = ip.split("[.]")
var ipNum = 0L
for (i <- 0 until fragments.length){
ipNum = fragments(i).toLong | ipNum << 8L
}
ipNum
}
//定义读取ip.txt规则,只要有用的数据
def readRules(path:String):Array[(Long,Long,String)] = {
//读取ip.txt
val bf: BufferedSource = Source.fromFile(path)
//对ip.txt进行整理
val lines: Iterator[String] = bf.getLines()
//对ip进行整理,并放入内存
val rules: Array[(Long, Long, String)] = lines.map(line => {
val fileds = line.split("[|]")
val startNum = fileds(2).toLong
val endNum = fileds(3).toLong
val province = fileds(6)
(startNum, endNum, province)
}).toArray
rules
}
//二分法查找
def binarySearch(lines: Array[(Long, Long, String)], ip: Long) : Int = {
var low = 0
var high = lines.length - 1
while (low <= high) {
val middle = (low + high) / 2
if ((ip >= lines(middle)._1) && (ip <= lines(middle)._2))
return middle
if (ip < lines(middle)._1)
high = middle - 1
else {
low = middle + 1
}
}
-1
}
def data2MySQL(it: Iterator[(String, Int)]): Unit = {
//一个迭代器代表一个分区,分区中有多条数据
//先获得一个JDBC连接
val conn: Connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/bigdata?characterEncoding=UTF-8", "root", "")
//将数据通过Connection写入到数据库
val pstm: PreparedStatement = conn.prepareStatement("INSERT INTO access_log VALUES (?, ?)")
//将分区中的数据一条一条写入到MySQL中
it.foreach(tp => {
pstm.setString(1, tp._1)
pstm.setInt(2, tp._2)
pstm.executeUpdate()
})
//将分区中的数据全部写完之后,在关闭连接
if(pstm != null) {
pstm.close()
}
if (conn != null) {
conn.close()
}
}
def main(args: Array[String]): Unit = {
//数据是在内存中
val rules: Array[(Long, Long, String)] = readRules("E:/Spark视频/小牛学堂-大数据24期-06-Spark安装部署到高级-10天/spark-04-Spark案例讲解/课件与代码/ip/ip.txt")
//将ip地址转换成十进制
val ipNum = ip2Long("1.24.6.56")
//查找
val index = binarySearch(rules,ipNum)
//根据脚标到rules中查找对应的数据
val tp = rules(index)
val province = tp._3
println(province)
}
}
Constant.scala
计算总的成交金额,定义保存redis中的key
package XXX
/**
* Create by 。。。
*
*/
object Constant {
val TOTAL_INCOME = "TOTAL_INCOME"
}