pyspark 第五章共享变量

from unittest import result
from pyspark import SparkConf,SparkContext
import json
# /opt/module/spark/bin/spark-submit /opt/Code/broadcast.py
if __name__ == '__main__':
    conf = SparkConf().setAppName("WorldCount").setMaster("local[*]")
    sc = SparkContext(conf=conf)
    stu_info_list = [(1,'张大仙',11),
                    (2,'王晓晓',13),
                    (3,'张甜甜',11),
                    (4,'王大力',11),
                    ]
    broadcast = sc.broadcast(stu_info_list)

    score_info_rdd = sc.parallelize([(1,'语文',99),
    (2,'语文',89),
    (3,'语文',79),
    (4,'语文',69)
    ])

    def map_func(data):
        id = data[0]
        name = ''
        value = broadcast.value
        for i in value:
            if id == i[0]:
                name = i[1]
        return (name,data[1],data[2])

    print(score_info_rdd.map(map_func).collect())

from unittest import result
from pyspark import SparkConf,SparkContext
import json
# /opt/module/spark/bin/spark-submit /opt/Code/broadcast.py
if __name__ == '__main__':
    conf = SparkConf().setAppName("WorldCount").setMaster("local[*]")
    sc = SparkContext(conf=conf)
    
    rdd = sc.parallelize([1,2,3,4,5,6,7,8,9,10],2)
    count = 0
    def map_func(data):
        global count
        count += 1
        print(count)
    rdd.map(map_func).collect()
    print(count)

from unittest import result
from pyspark import SparkConf,SparkContext
import json
# /opt/module/spark/bin/spark-submit /opt/Code/broadcast.py
if __name__ == '__main__':
    conf = SparkConf().setAppName("WorldCount").setMaster("local[*]")
    sc = SparkContext(conf=conf)
    
    rdd = sc.parallelize([1,2,3,4,5,6,7,8,9,10],2)
    acmlt = sc.accumulator(0)
    def map_func(data):
        global acmlt
        acmlt += 1
        print(acmlt)
    rdd.map(map_func).collect()
    print(acmlt)

from unittest import result
from pyspark import SparkConf,SparkContext
import json
# /opt/module/spark/bin/spark-submit /opt/Code/broadcast.py
if __name__ == '__main__':
    conf = SparkConf().setAppName("WorldCount").setMaster("local[*]")
    sc = SparkContext(conf=conf)
    
    rdd = sc.parallelize([1,2,3,4,5,6,7,8,9,10],2)
    acmlt = sc.accumulator(0)
    def map_func(data):
        global acmlt
        acmlt += 1
        print(acmlt)
    rdd2 = rdd.map(map_func)
    rdd2.cache()
    rdd2.collect()
    rdd3 = rdd2.map(lambda x:x)
    rdd3.collect()
    print(acmlt)

from unittest import result
from pyspark import SparkConf,SparkContext
import json
import re
from operator import add
# /opt/module/spark/bin/spark-submit /opt/Code/broadcast.py
if __name__ == '__main__':
    conf = SparkConf().setAppName("WorldCount").setMaster("local[*]")
    sc = SparkContext(conf=conf)
    
    file_rdd = sc.textFile('file:///opt/Data/accumulator_broadcast_data.txt')
    line_rdd = file_rdd.filter(lambda line:line.strip()) # 过滤空行
    data_rdd = line_rdd.map(lambda x:x.strip()) # 将前后空格去除
    words_rdd = data_rdd.flatMap(lambda x:re.split('\s+',x))

    abnormal_char = [',','.','!','#','$','%']
    broadcast = sc.broadcast(abnormal_char)
    acmlt = sc.accumulator(0)

    def filter_func(data):
        global acmlt 
        abnormal_char = broadcast.value
        if data in abnormal_char:
            acmlt += 1
            return False
        else:
            return True
    
    normal_words_rdd =words_rdd.filter(filter_func)
    result_rdd = normal_words_rdd.map(lambda x: (x,1)).reduceByKey(add)
    print('正常单词计数结果: ',result_rdd.collect())
    print('特殊字符数量: ',acmlt)

相关资料：
黑马全网首发PySpark视频教程
链接：https://pan.baidu.com/s/1AY7FmO6w-jCAj7t_tD9oGw
提取码：1234