SparkStreaming03

SparkStreaming03

高德地图逆地理转码

package com.hpznyf.HTTPApp

import com.alibaba.fastjson.JSON
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils

object HttpClientApp {
  def main(args: Array[String]): Unit = {

    val longtitude = 116.480881
    val latitude = 39.989410

    val key = ""
    val url = s"https://restapi.amap.com/v3/geocode/regeo?key=${key}&location=$longtitude,$latitude"

    val httpClient = HttpClients.createDefault()
    val httpGet = new HttpGet(url)
    val response = httpClient.execute(httpGet)


    try{
      val status = response.getStatusLine.getStatusCode
      val entity = response.getEntity

      if(status == 200 ){
        val result = EntityUtils.toString(entity)
        val json = JSON.parseObject(result)

        val regeocode = json.getJSONObject("regeocode")
        if(regeocode != null){
          val addressComponent = regeocode.getJSONObject("addressComponent")
          val province = addressComponent.getString("province")
          println(province)
        }


      }
    }catch {
      case e : Exception => e.printStackTrace()
    }
  }
}

sparkstreaming 解析ip

import com.alibaba.fastjson.JSON
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.{CloseableHttpClient, HttpClients}
import org.apache.http.util.EntityUtils
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.sql.SparkSession
import org.apache.spark.streaming.{Seconds, StreamingContext}

/**
 * pk,111111,116.480881,39.989410
 * ruoze,22222,116.480881,39.989410
 */
object HttpStreamingApp extends Logging{
  def main(args: Array[String]): Unit = {
    val sparkconf = new SparkConf().setAppName(this.getClass.getCanonicalName).setMaster("local[2]")
    val ssc = new StreamingContext(sparkconf, Seconds(5))

    val lines = ssc.socketTextStream("hadoop001", 9527)

    lines.foreachRDD(rdd => {
      if(!rdd.isEmpty()){
        rdd.foreachPartition(partition => {

          //开启连接
          val httpClient = HttpClients.createDefault()
          logError("----------------" + httpClient)

          //数据清洗功能
          partition.foreach( x => {
            val splits = x.split(",")
            val userId = splits(0)
            val time = splits(1)
            val longtitude = splits(2).toDouble
            val latitude = splits(3).toDouble

            val key = ""
            val url = s"https://restapi.amap.com/v3/geocode/regeo?key=${key}&location=$longtitude,$latitude"

            var province = ""
            var regeocode:CloseableHttpClient = null
            try{
              val httpGet = new HttpGet(url)
              val response = httpClient.execute(httpGet)

              val status = response.getStatusLine.getStatusCode
              val entity = response.getEntity

              if(status == 200 ){
                val result = EntityUtils.toString(entity)
                val json = JSON.parseObject(result)

                val regeocode = json.getJSONObject("regeocode")
                if(regeocode != null){
                  val addressComponent = regeocode.getJSONObject("addressComponent")
                  val province = addressComponent.getString("province")
                  logError("/// " + province)
                }
              }
            }catch {
              case e : Exception => e.printStackTrace()
            }finally {
              if(regeocode != null){
                regeocode.close()
              }
            }
          })
        })
      }
    })

    ssc.start()
    ssc.awaitTermination()
  }
}


case class Access(userId:String, time:String, longtitude:Double, latitude:Double, province:String)

updateStateBykey

object StateApp {
  def main(args: Array[String]): Unit = {
    val sparkconf = new SparkConf().setAppName(this.getClass.getCanonicalName).setMaster("local[2]")
    val ssc = new StreamingContext(sparkconf, Seconds(5))
    ssc.checkpoint("data/checkpoint")
    val lines = ssc.socketTextStream("hadoop001", 9527)

    val result = lines.flatMap(_.split(","))
      .map((_,1)).updateStateByKey(updateFunction)

    result.print()

    ssc.start()
    ssc.awaitTermination()
  }

  def updateFunction(newValue: Seq[Int], preValues: Option[Int]) :Option[Int] ={

    val newCount = newValue.sum
    val pre = preValues.getOrElse(0)

    Some(newCount + pre)
  }
}

mapWithState

  def main(args: Array[String]): Unit = {
    val sparkconf = new SparkConf().setAppName(this.getClass.getCanonicalName).setMaster("local[2]")
    val ssc = new StreamingContext(sparkconf, Seconds(5))
    ssc.checkpoint("data/checkpoint")
    val lines = ssc.socketTextStream("hadoop001", 9527)

    val result = lines.flatMap(_.split(","))
      .map((_,1)).mapWithState(
      StateSpec.function(mappingFunc).timeout(Seconds(100))
    )

    result.print()

    ssc.start()
    ssc.awaitTermination()
  }
  val mappingFunc = (word: String, one:Option[Int], state:State[Int]) => {
    val sum = one.getOrElse(0) + state.getOption().getOrElse(0)
    val output = (word, sum)
    state.update(sum)
    output
  }

redis

redis.conf

port 16379
bind 0.0.0.0
	daemonize yes
	
	
	databases 16  不用改