Dstream有状态转化操作（updateStateByKey）

#!/usr/bin/evn python3
from __future__ import print_function
import sys
from pyspark import SparkContext
from pyspark.streaming import StreamingContext
import pymysql

if __name__ == "__main__":
    if len(sys.argv) != 3:
        print("Usage：NetworkWordCountStateful.py<hostname><port>",file=sys.stderr)
        exit(-1)
    sc = SparkContext(appName="PythonSteamingStreamingStatefulNetworkWordCount")
    ssc = StreamingContext(sc, 1)
    # 创建一个检查点
    ssc.checkpoint("file:///usr/local/spark/mycode/streaming/stateful/")
    # 初始RDD中有添加两个元素
    initialStateRDD = sc.parallelize(([u'hello', 1], [u'world', 1]))

    # 跨批次状态累加
    def updateFunc(new_values, last_num):
        return sum(new_values)+(last_num or 0)
    #    创建套接字流
    lines = ssc.socketTextStream(sys.argv[1],int(sys.argv[2]))
    # 进行map
    running_counts = lines.flatMap(lambda line:line.split(" ")).map(lambda word: (word, 1))\
                    .updateStateByKey(updateFunc,initialRDD=initialStateRDD)

    # 保存在文本文件中
    # running_counts.saveAsTextFiles("file:///usr/local/spark/mycode/streaming/stateful/")
    # running_counts.pprint()

    def dbfunc(records):
        db = pymysql.connect("localhost", "root", "123456", "spark")
        cursor = db.cursor()

        def doinsert(p):
            sql = '''insert into wordcount(word, count) values('%s', '%s')''' % (str(p[0]), str(p[1]))
            try:
                cursor.execute(sql)
                db.commit()
            except:
                db.rollback()
            for item in records:
                doinsert(item)

    def func(rdd):
        repartitionedRDD = rdd.repartition(3)
        repartitionedRDD.foreachPatrtition(dbfunc)

    running_counts.foreachRDD(func)
    ssc.start()
    ssc.awaitTermination()

学习链接：https://study.163.com/course/courseLearn.htm?courseId=1209408816#/learn/text?lessonId=1279273524&courseId=1209408816