需求:SparkStreaming实时写入Hive
关于怎么写,网上一大堆,我简单点列下代码:
SparkConf sparkConf = new SparkConf().setAppName("sparkStreaming-order").setMaster(SPARK_MASTER);
sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.set("spark.streaming.kafka.maxRatePerPartition", "500")
.set("spark.kryo.registrator", "com.ykc.bean.input.MyRegistrator") //序列化ConsumerRecord类
.set("hive.metastore.uris", HIVE_METASTORE_URIS)
.set("spark.sql.warehouse.dir", HIVE_WAREHOUSE_DIR)
.set("hive.exec.dynamic.partition", "true")
.set("hive.exec.max.dynamic.partitions", "2048")
.set("hive.exec.dynamic.partition.mode", "nonstrict");
SparkSession ss = SparkSession.builder().config(sparkConf).enableHiveSupport().getOrCreate();
JavaStreamingContext jsc = new JavaStreamingContext(new JavaSparkContext(ss.sparkContext()), Durations.seconds(30));
// 注意这里有个问题,orderStream在最外面定义它为static或transient,原因在sparkStreaming使用sql这篇博客中有讲过
// 还有手动提交offset,前面也有提过
orderStream = KafkaUtils.createDirectStream(jsc, LocationStrategies.PreferConsistent(),ConsumerStrategies.Subscribe(Lists.newArrayList(topic), kafkaParams, getOffsets(topic)));
// checkpoint
jsc.checkpoint(SPARK_CHECKPOINT_DIR + "/order");
orderStream.checkpoint(Durations.seconds(SPARK_CHECKPOINT_INTERVAL));
orderStream.foreachRDD(new VoidFunction<JavaRDD<ConsumerRecord<String, String>>>() {
private static final long serialVersionUID = 1L;
@Override
public v