sparkStreaming+flume

spark ***********************

import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.SparkConf
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.flume._

object flume extends App{
  val sparkConf = new SparkConf().setMaster("local")
    .setAppName("My App")
  val ssc = new StreamingContext(sparkConf, Seconds(1))

  // Create a flume stream
  val stream = FlumeUtils.createStream(ssc, "192.168.162.44", 11000, StorageLevel.MEMORY_ONLY_SER_2)
  stream.map( _.event.getBody.array() ).map( new String(_) ).print()

  // Print out the count of events received from this server in each batch
  stream.count().map(cnt => "Received " + cnt + " flume events." ).print()
  ssc.start()
  ssc.awaitTermination()
}

flume  agent1 **********************************

#agent1 name
agent1.channels = c1
agent1.sources = r1
agent1.sinks = k1

#set channel
agent1.channels.c1.type = memory
agent1.channels.c1.capacity = 1000
agent1.channels.c1.transactionCapacity = 100

agent1.sources.r1.channels = c1
agent1.sources.r1.type = exec
agent1.sources.r1.command = tail -F /home/hadoop/work/log/access.log

agent1.sources.r1.interceptors = i1 i2
agent1.sources.r1.interceptors.i1.type = static
agent1.sources.r1.interceptors.i1.key = Type
agent1.sources.r1.interceptors.i1.value = LOGIN
agent1.sources.r1.interceptors.i2.type = timestamp

# set sink1
agent1.sinks.k1.channel = c1
agent1.sinks.k1.type = avro
agent1.sinks.k1.hostname = ************
agent1.sinks.k1.port = 11000

* 启动 先启动spark streaming

然后 bin/flume-ng agent -n agent1 -c conf -f conf/agent1.conf -Dflume.root.logger=INFO,console