学习笔记--python(7、pyspark)

1、数据输入

from pyspark import SparkConf,SparkContext

conf=SparkConf().setMaster("local[*]").setAppName("test_spark")
sc=SparkContext(conf=conf)
# 通过parallelize方法将python对象加入spark内，成为rdd对象
rdd1=sc.parallelize([1,2,3,4,5,6,7])
rdd2=sc.parallelize((1,2,3,4))
rdd3=sc.parallelize("abcdefg")
rdd4=sc.parallelize({1,2,3,4,5})
rdd5=sc.parallelize({"key1":"value1","key2":"value2"})
# 通过collect()方法查看rdd里面的内容
print(rdd1.collect())
print(rdd2.collect())
print(rdd3.collect())
print(rdd4.collect())
print(rdd5.collect())

# 读取文件
rdd=sc.textFile("D:/ab.txt")
print(rdd.collect())
sc.stop()

2、数据计算

 

# Map方法
from pyspark import SparkConf,SparkContext
# 标明python解释器的位置
import os
os.environ['PYSPARK_PYTHON'] = "C:/Users/86159/AppData/Roaming/Python/Python312"
conf = SparkConf().setMaster("local[*]").setAppName("test_spark")
sc = SparkContext(conf=conf)
# 准备一个rdd
rdd= sc.parallelize([1,3,5])
# 使用map方法将数据加上6
# def func(data):
#     return data+6
# rdd2=rdd.map(func)
rdd2 = rdd.map(lambda x: x*10).map(lambda x:x*2)
print(rdd2.collect())


#flatMap方法
from pyspark import SparkConf,SparkContext
# 标明python解释器的位置
import os
os.environ['PYSPARK_PYTHON'] = "C:/Users/86159/AppData/Roaming/Python/Python312"
conf = SparkConf().setMaster("local[*]").setAppName("test_spark")
sc = SparkContext(conf=conf)
# 准备rdd
rdd=sc.parallelize(["123","456","789"])
# 将每个字符提取出来
rdd2=rdd.flatMap(lambda x:x.split(" "))
print(rdd2.collect())


# redudeByKey方法
from pyspark import SparkConf,SparkContext
# 标明python解释器的位置
import os
os.environ['PYSPARK_PYTHON'] = "C:/Users/86159/AppData/Roaming/Python/Python312"
conf = SparkConf().setMaster("local[*]").setAppName("test_spark")
sc = SparkContext(conf=conf)
rdd=sc.parallelize([('1',23),('2',34),('3',90),('2',67)])
rdd2=rdd.reduceByKey(lambda a,b:a+b)
print(rdd2.collect())


# fliter方法
from pyspark import SparkConf,SparkContext
# 标明python解释器的位置
import os
os.environ['PYSPARK_PYTHON'] = " "# python解释器的位置
conf = SparkConf().setMaster("local[*]").setAppName("test_spark")
sc = SparkContext(conf=conf)
rdd=sc.parallelize([1,2,3,4,5,6])
# 过滤筛选
rdd2=rdd.filter(lambda num:num%2==0)
print(rdd2.collect())


#distinct方法
from pyspark import SparkConf,SparkContext
# 标明python解释器的位置
import os
os.environ['PYSPARK_PYTHON'] = " "# python解释器的位置
conf = SparkConf().setMaster("local[*]").setAppName("test_spark")
sc = SparkContext(conf=conf)
# 去重
rdd=sc.parallelize([1,2,3,4,5,3,22,1])
rdd2=rdd.distinct()
print(rdd2.collect()) 


# sortBy方法
from pyspark import SparkConf,SparkContext
# 标明python解释器的位置
import os
os.environ['PYSPARK_PYTHON'] = ""  # python解释器的位置
conf = SparkConf().setMaster("local[*]").setAppName("test_spark")
sc = SparkContext(conf=conf)
rdd=sc.textFile("D:/hello.txt")
word_rdd=rdd.flatMap(lambda x:x.split(" "))
word_with_one_rdd=word_rdd.map(lambda word:(word,1))
result_rdd=word_with_one_rdd.reduceByKey(lambda a,b:a+b)
# 对结果进行排序
# False 为从大到小，全局排序设置分区数为1
final_result_rdd=result_rdd.sortBy(lambda x:x[1],ascending=False,numPartitions=1)
print(final_result_rdd.collect())

3、数据输出为python对象

from pyspark import SparkConf,SparkContext
# 标明python解释器的位置
import os
os.environ['PYSPARK_PYTHON'] = " "# python解释器的位置
conf = SparkConf().setMaster("local[*]").setAppName("test_spark")
sc = SparkContext(conf=conf)
rdd=sc.parallelize([1,2,3,4,5,6])

# collect算子，输出rdd为list对象
rdd_list:list=rdd.collect()
print(rdd_list)
print(type(rdd_list))

# reduce算子，对rdd进行两两聚合
num=rdd.reduce(lambda a,b:a+b)
print(num)

# take算子，取出rdd前N个元素，组成list返回
take_list=rdd.take(3)
print(take_list)

# count算子，计算rdd内有多少条数据，返回值为数字
num_count=rdd.count()
print(f"共有元素{num_count}个")
sc.stop()

4、数据输出到文件中

from pyspark import SparkConf,SparkContext
# 标明python解释器的位置
import os
os.environ['PYSPARK_PYTHON'] = " "python解释器位置
conf = SparkConf().setMaster("local[*]").setAppName("test_spark")
sc = SparkContext(conf=conf)
rdd1=sc.parallelize([1,2,3,4,5,6,7])
rdd2=sc.parallelize(["年",2],["月",8],["日",7])
rdd3=sc.parallelize([[1,2,3],[3,2,1][1,4,7]])
rdd1.saveAsTextFile("D:/output1")
rdd2.saveAsTextFile("D:/output2")
rdd3.saveAsTextFile("D:/output3")