Spark Low level API RDD学习笔记:key_value RDD

常规操作

# 创建
myCollection = "Spark The Definitive Guide : Big Data Processing Made Simple"\
.split(" ")
words = spark.sparkContext.parallelize(myCollection, 2)

keyword = words.keyBy(lambda word:word.lower()[0])
keyword.mapValues(lambda word: word.upper()).collect()
[('s', 'SPARK'),
('t', 'THE'),
('d', 'DEFINITIVE'),
('g', 'GUIDE'),
(':', ':'),
('b', 'BIG'),
('d', 'DATA'),
('p', 'PROCESSING'),
('m', 'MADE'),
('s', 'SIMPLE')]

# look up the result for particular key
keyword.lookup("s")
['Spark', 'Simple']

#sampleByKey
#sample an RDD by a set of keys
#RDD.sampleByKey(withReplacement, fractions, seed=None)[source]
# 第一个是是否有放回，第二个是概率，第三个是随机数种子
# 这个没法确认返回子集的大小
import random
## extract characters in words
distinctChars = words.flatMap(lambda word:list(word.lower())\
                     .distinct()\
                     .collect() 
 
sampleMap = dict(map(lambda c:(c, random.random()), distinctChars))
words.map(lambda word: (word.lower()[0], word))\
    .sampleByKey(True, sampleMap, 6)\
    .collect()