pyspark-Transform

pyspark之Transform

由于工作中要用到spark，写scala还是太菜，趁机学学pyspark这个神器
记录这个主要是自己以后查看方便，其中基本上参考3万字长文，PySpark入门级学习教程，其中也有自己增加了部分算子

# -- coding: utf-8 --
# Author  : ming
# time: 2022-03-08

import findspark
findspark.init()

import os
import pyspark
from pyspark import SparkConf, SparkContext

conf = SparkConf().setAppName("transform_function").setMaster("local[*]")
sc  = SparkContext(conf=conf)

"""
Transform 算子解析
"""

# 1， map
rdd1 = sc.parallelize(range(1, 11), 4)
rdd1_map = rdd1.map(lambda x: x*x)
print(rdd1_map.collect())

# 2. flatMap
rdd2 = sc.parallelize(["hello ming", "hello spark", "welcome hadoop", "bye scala"])
rdd2_flat_map = rdd2.flatMap(lambda x: x.split(" "))
print(rdd2_flat_map.collect())

# 3. filter
rdd3 = sc.parallelize(range(1, 11), 4)
print(rdd3.collect())
rdd3_filter = rdd3.filter(lambda x: x % 2 == 0)
print(rdd3_filter.collect())

# 4. distinct
rdd4 = sc.parallelize([1, 2, 2, 3, 3, 3, 4, 5, 6, 6, 7, 8])
rdd4_distinct = rdd4.distinct()
print(rdd4_distinct.collect())

# 5. reduceByKey
from operator import add
rdd5 = sc.parallelize([("a", 1), ("b", 1), ("a", 3)])
rdd5_reduce_by_key = rdd5.reduceByKey(add)
print(rdd5_reduce_by_key.collect())

# 6. mapPartitions: 根据分区内的数据进行映射操作
rdd6 = sc.parallelize([1, 2, 3, 4], 2)
def f(iterator):
    yield sum(iterator)
rdd6_map_partitions = rdd6.mapPartitions(f)
print(rdd6_map_partitions.collect())

# 7. mapPartitionsWithIndex ： 对分区的index进行操作
rdd7 = sc.parallelize([1, 2, 3, 4], 4)
def f(splitIndex, iter):
    yield splitIndex
rdd7_map_partition_with_index = rdd7.mapPartitionsWithIndex(f).sum()
print(rdd7_map_partition_with_index)

# 8. sortBy: 根据规则进行排序
rdd8 = sc.parallelize([("a", 1), ("b", 2), ("1", 7), ("d", 4), ("f", 3)])
rdd8_sort_by_key = rdd8.sortBy(lambda x: x[0])
print(rdd8_sort_by_key.collect())
rdd8_sort_by_value = rdd8.sortBy(lambda x: x[1])
print(rdd8_sort_by_value.collect())

# 9. subtract: 数据集相减
x = sc.parallelize([("a", 1), ("b", 4), ("b", 5), ("a", 3)])
y = sc.parallelize([("a", 3), ("c", None)])
rdd9 = x.subtract(y)
print(rdd9.collect())
rdd9_sorted = sorted(x.subtract(y).collect())
print(rdd9_sorted)

# 10. intersection : 取两个RDD的交集，同时有去重的效果
rdd10_1 = sc.parallelize([1,2,4,6,7,8,7,4,8,8,9])
rdd10_2 = sc.parallelize([1,2,3,5,7,8,8,9,10])
rdd10_intersection = rdd10_1.intersection(rdd10_2)
print(rdd10_intersection.collect())

# 11 union:
rdd11_1 = sc.parallelize([1,2,3,4])
rdd11_2 = sc.parallelize([1,4,5,3,6])
rdd11_union = rdd11_1.union(rdd11_2)
print(rdd11_union.collect())

# 12. 笛卡尔积
rdd12_1 = sc.parallelize([1,2,3])
rdd12_2 = sc.parallelize([1,2])
rdd12_cartesian = rdd12_1.cartesian(rdd12_2)
print(rdd12_cartesian.collect())

# 13. zip: 拉链合并， 需要两个RDD具有相同的长度和分区数量
rdd13_1 = sc.parallelize(range(10))
rdd13_2 = sc.parallelize(range(1000, 1010))
rdd13_zip = rdd13_1.zip(rdd13_2)
print(rdd13_zip.collect())

# 14. zipWithIndex: 将rdd和一个从0开始的递增序列按照拉链连接
rdd14 = sc.parallelize(["hello", "spark", "python", "hadoop", "scala"])
rdd14_zip_with_index = rdd14.zipWithIndex()
print(rdd14_zip_with_index.collect())

# 15. groupByKey
rdd15 = sc.parallelize([("a", 1), ("b", 2), ("c", 3), ("b", 4)])
rdd15_groupby_key = rdd15.groupByKey().mapValues(max)
print(rdd15_groupby_key.collect())

# 16. sortByKey
rdd16 = sc.parallelize([("a", 1), ("b", 2), ("1", 3), ("d", 4)])
rdd16_sort_by_key = rdd16.sortByKey()
print(rdd16_sort_by_key.collect())

# 17. join
rdd17_1 = sc.parallelize([("a", 1), ("b", 4)])
rdd17_2 = sc.parallelize([("a",2), ("a", 3)])
rdd17_join = sorted(rdd17_1.join(rdd17_2).collect())
print(rdd17_join)

# 18. leftOuterJoin
rdd18_1 = sc.parallelize([("a", 1), ("b", 4)])
rdd18_2 = sc.parallelize([("a", 3)])
rdd18_left_outer_join = rdd18_1.leftOuterJoin(rdd18_2)
print(rdd18_left_outer_join.collect())
rdd18_right_outer_join = rdd18_1.rightOuterJoin(rdd18_2)
print(rdd18_right_outer_join.collect())