spark stream 应用-结合kafka

本文章主要通过spark streaming结合kafka的两种读取数据形式实现wordcount统计

1 基于kafka direct方式实时统计word

import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import kafka.serializer.StringDecoder;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaPairInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.kafka.KafkaUtils;

import scala.Tuple2;

/**
 * 基于Kafka Direct方式的实时wordcount程序
 * @author Administrator
 *
 */
public class KafkaDirectWordCount {

   public static void main(String[] args) {
      SparkConf conf = new SparkConf()
            .setMaster("local[2]")
            .setAppName("KafkaDirectWordCount");  
      JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(5));
      
      // 首先，要创建一份kafka参数map
      Map<String, String> kafkaParams = new HashMap<String, String>();
      kafkaParams.put("metadata.broker.list", 
            "192.168.1.107:9092,192.168.1.108:9092,192.168.1.109:9092");
      
      // 然后，要创建一个set，里面放入，你要读取的topic
      // 这个，就是我们所说的，它自己给你做的很好，可以并行读取多个topic
      Set<String> topics = new HashSet<String>();
      topics.add("WordCount");
      
      // 创建输入DStream
      JavaPairInputDStream<String, String> lines = KafkaUtils.createDirectStream(
            jssc, 
            String.class, 
            String.class, 
            StringDecoder.class, 
            StringDecoder.class, 
            kafkaParams, 
            topics);

      // 执行wordcount操作
      JavaDStream<String> words = lines.flatMap(
            
            new FlatMapFunction<Tuple2<String,String>, String>() {

               private static final long serialVersionUID = 1L;

               @Override
               public Iterable<String> call(Tuple2<String, String> tuple)
                     throws Exception {
                  return Arrays.asList(tuple._2.split(" "));  
               }
               
            });
      
      JavaPairDStream<String, Integer> pairs = words.mapToPair(
            
            new PairFunction<String, String, Integer>() {

               private static final long serialVersionUID = 1L;

               @Override
               public Tuple2<String, Integer> call(String word) throws Exception {
                  return new Tuple2<String, Integer>(word, 1);
               }
               
            });
      
      JavaPairDStream<String, Integer> wordCounts = pairs.reduceByKey(
            
            new Function2<Integer, Integer, Integer>() {

               private static final long serialVersionUID = 1L;

               @Override
               public Integer call(Integer v1, Integer v2) throws Exception {
                  return v1 + v2;
               }
               
            });
      
      wordCounts.print();
      
      jssc.start();
      jssc.awaitTermination();
      jssc.close();
   }
   
}

2 基于kafka receiver方式实时统计word

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaPairReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.kafka.KafkaUtils;

import scala.Tuple2;

/**
 * 基于Kafka receiver方式的实时wordcount程序
 * @author Administrator
 *
 */
public class KafkaReceiverWordCount {

   public static void main(String[] args) {
      SparkConf conf = new SparkConf()
            .setMaster("local[2]")
            .setAppName("KafkaWordCount");  
      JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(5));
      
      // 使用KafkaUtils.createStream()方法，创建针对Kafka的输入数据流
      Map<String, Integer> topicThreadMap = new HashMap<String, Integer>();
      topicThreadMap.put("WordCount", 1);
      
      JavaPairReceiverInputDStream<String, String> lines = KafkaUtils.createStream(
            jssc, 
            "192.168.1.107:2181,192.168.1.108:2181,192.168.1.109:2181", 
            "DefaultConsumerGroup", 
            topicThreadMap);
      
      // 然后开发wordcount逻辑
      JavaDStream<String> words = lines.flatMap(
            
            new FlatMapFunction<Tuple2<String,String>, String>() {

               private static final long serialVersionUID = 1L;

               @Override
               public Iterable<String> call(Tuple2<String, String> tuple)
                     throws Exception {
                  return Arrays.asList(tuple._2.split(" "));  
               }
               
            });
      
      JavaPairDStream<String, Integer> pairs = words.mapToPair(
            
            new PairFunction<String, String, Integer>() {

               private static final long serialVersionUID = 1L;

               @Override
               public Tuple2<String, Integer> call(String word)
                     throws Exception {
                  return new Tuple2<String, Integer>(word, 1);
               }
               
            });
      
      JavaPairDStream<String, Integer> wordCounts = pairs.reduceByKey(
            
            new Function2<Integer, Integer, Integer>() {
         
               private static final long serialVersionUID = 1L;

               @Override
               public Integer call(Integer v1, Integer v2) throws Exception {
                  return v1 + v2;
               }
               
            });
      
      wordCounts.print();  
      
      jssc.start();
      jssc.awaitTermination();
      jssc.close();
   }
   
}

总结：

Direct：直接依赖于kafka自身api

这种方式是延迟的。也就是说当action真正触发时才会去kafka里接数据。因此不存在currentbuffer的概念。它把kafka每个分区里的数据，映射为KafkaRdd的概念。题外话，在structured streaming中，也已经向DataFrame和DataSet统一了，弱化了RDD的概念。

真正与kafka打交道的是KafkaCluster,全限定名： org.apache.spark.streaming.kafka.KafkaCluster。包括设备kafka各种参数，连接，获取分区，以及偏移量，设置偏移量范围等。

Direct方式采用Kafka简单的consumer api方式来读取数据，无需经由ZooKeeper，此种方式不再需要专门Receiver来持续不断读取数据。当batch任务触发时，由Executor读取数据，并参与到其他Executor的数据计算过程中去。driver来决定读取多少offsets，并将offsets交由checkpoints来维护。将触发下次batch任务，再由Executor读取Kafka数据并计算。从此过程我们可以发现Direct方式无需Receiver读取数据，而是需要计算时再读取数据，所以Direct方式的数据消费对内存的要求不高，只需要考虑批量计算所需要的内存即可；另外batch任务堆积时，也不会影响数据堆积

Receiver：依赖于zookeeper

Spark官方最先提供了基于Receiver的Kafka数据消费模式。不过这种方式是先把数据从kafka中读取出来，然后缓存在内存，再定时处理。如果这时候集群退出，而偏移量又没处理好的话，数据就丢掉了，存在程序失败丢失数据的可能，后在Spark 1.2时引入一个配置参数spark.streaming.receiver.writeAheadLog.enable以规避此风险。

Receiver-based的Kafka读取方式是基于Kafka高阶(high-level) api来实现对Kafka数据的消费。在提交Spark Streaming任务后，Spark集群会划出指定的Receivers来专门、持续不断、异步读取Kafka的数据，读取时间间隔以及每次读取offsets范围可以由参数来配置。读取的数据保存在Receiver中，具体StorageLevel方式由用户指定，诸如MEMORY_ONLY等。当driver 触发batch任务的时候，Receivers中的数据会转移到剩余的Executors中去执行。在执行完之后，Receivers会相应更新ZooKeeper的offsets。如要确保at least once的读取方式，可以设置spark.streaming.receiver.writeAheadLog.enable为true