SparkStreaming整合Flume --push方式与--pull方式

 test-flume-push-streaming.conf

#flume-push-streaming

flume-push-streaming.sources = netcat-source
flume-push-streaming.sinks = avro-sink
flume-push-streaming.channels = memory-channel

flume-push-streaming.sources.netcat-source.type = netcat
flume-push-streaming.sources.netcat-source.bind = 192.168.137.100
flume-push-streaming.sources.netcat-source.port = 44444

#flume把数据写到什么机器的什么端口
#这里的192.168.72.59是用IDEA运行spark streaming程序的一台电脑

flume-push-streaming.sinks.avro-sink.type = avro
flume-push-streaming.sinks.avro-sink.hostname = 192.168.72.59
flume-push-streaming.sinks.avro-sink.port = 41414

flume-push-streaming.channels.memory-channel.type = memory

flume-push-streaming.sources.netcat-source.channels = memory-channel
flume-push-streaming.sinks.avro-sink.channel = memory-channel

 

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.sid.spark</groupId>
    <artifactId>spark-train</artifactId>
    <version>1.0</version>
    <inceptionYear>2008</inceptionYear>
    <properties>
        <scala.version>2.11.8</scala.version>
        <kafka.version>0.9.0.0</kafka.version>
        <spark.version>2.2.0</spark.version>
        <hadoop.version>2.9.0</hadoop.version>
        <hbase.version>1.4.4</hbase.version>
    </properties>

    <repositories>
        <repository>
            <id>scala-tools.org</id>
            <name>Scala-Tools Maven2 Repository</name>
            <url>http://scala-tools.org/repo-releases</url>
        </repository>
    </repositories>

    <pluginRepositories>
        <pluginRepository>
            <id>scala-tools.org</id>
            <name>Scala-Tools Maven2 Repository</name>
            <url>http://scala-tools.org/repo-releases</url>
        </pluginRepository>
    </pluginRepositories>

    <dependencies>

        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scala.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka_2.11</artifactId>
            <version>${kafka.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

        <!--<dependency>-->
        <!--<groupId>org.apache.hbase</groupId>-->
        <!--<artifactId>hbase-clinet</artifactId>-->
        <!--<version>${hbase.version}</version>-->
        <!--</dependency>-->
        <!--<dependency>-->
        <!--<groupId>org.apache.hbase</groupId>-->
        <!--<artifactId>hbase-server</artifactId>-->
        <!--<version>${hbase.version}</version>-->
        <!--</dependency>-->

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>

        <dependency>
            <groupId>net.jpountz.lz4</groupId>
            <artifactId>lz4</artifactId>
            <version>1.3.0</version>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.31</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming-flume_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
    </dependencies>
    <build>
        <sourceDirectory>src/main/scala</sourceDirectory>
        <!--<testSourceDirectory>src/test/scala</testSourceDirectory>-->
    </build>
</project>

代码：

package com.zoujc.sparkstreaming

import org.apache.spark.SparkConf
import org.apache.spark.streaming.flume.FlumeUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}

/**
  * SparkStreaming整合Flume用Push方式 模拟词频统计 无状态
  */
object FlumePushSparkStreaming {
   def main(args: Array[String]): Unit = {

      //SparkStreaming从什么机器的端口接收数据
      if(args.length != 2){
         System.err.println("Usage: FlumePushStreaming <hostname> <port>")
         System.exit(1)
      }
      val Array(hostanme,port) = args

      //本地运行
      val sparkConf = new SparkConf().setAppName("FlumePushSparkStreaming").setMaster("local[2]")
      val ssc = new StreamingContext(sparkConf,Seconds(2))



      //该代码运行在192.168.72.59机器上，flume会把数据push到这台机器上来
      val flumeStream = FlumeUtils.createStream(ssc,hostanme,port.toInt)
      //flume在传输的时候数据event是有head和body的，这里只拿body就好了
      flumeStream.map(x => new String(x.event.getBody.array()).trim)
              .flatMap(_.split(" ")).map((_,1)).reduceByKey(_ + _).print()

      ssc.start()
      ssc.awaitTermination()

   }
}

//这是pull方式只需改动一行

val flumeStream = FlumeUtils.createPollingStream(ssc, hostname, port.toInt)

启动IDEA项目 传入参数hostname port

启动192.168.137.100的agent

./flume-ng agent --name flume-push-streaming --conf $FLUME_HOME/conf --conf-file /root/flume/test-flume-push-streaming.conf -Dflume.root.logger=INFO,console

在192.168.137.100中

telnet node1 44444

因为flume数据的来源配置的是从192.168.137.100的44444端口接收数据

发送数据：

在控制台显示：

词频统计------------------------------

 

本地运行通过以后打包传到服务器上运行：

修改代码：

打包

把target下面的jar包传到服务器,提交到spark上运行

./spark-submit --class com.zoujc.sparkstreaming.FlumePushSparkStreaming --master local[2] --name FlumePushSparkStreaming --packages org.apache.spark:spark-streaming-flume_2.11:2.2.0 /root/lib/spark-train-1.0.jar 192.168.137.100 41414

修改flume的sink地址到192.168.137.100，重启flume

启动192.168.137.100的agent

./flume-ng agent --name flume-push-streaming --conf $FLUME_HOME/conf --conf-file /root/flume/test-flume-push-streaming.conf -Dflume.root.logger=INFO,console

telnet 192.168.137.100 44444

然后查看服务器运行的spark