spark python初学（一）针对某个单词计数版本0

初学spark 还有很多不会的。这是针对spark下README.md进行单词计数的python代码。

这段代码主要计数的对象是单词”Spark“ 和”spark“

# -*- coding:UTF-8 -*-

from pyspark import SparkContext, SparkConf


# 读取文件并处理
def load_and_deal(file_path):
    inputRDD = sc.textFile(file_path)
    SparkRdd = inputRDD.filter(lambda s: "Spark" in s)
    sparkRdd = inputRDD.filter(lambda s: "spark" in s)
    allRdd = SparkRdd.union(sparkRdd)
    disRdd = allRdd.distinct()
    words = disRdd.flatMap(lambda x: x.split(" "))
    result = words.map(lambda x: (x, 1)).reduceByKey(lambda x, y: x+y)
    return result

# 计数
def count_words(rdd):
    Sparknum = 0
    sparknum = 0
    for item in rdd.collect():
        print(item)  # 包含Spark和saprk的所有句子中的单词计数
        print item[0].count("spark")

        if "Spark" in item[0]:
            Sparknum += item[1]  # Spark计数
        if "spark" in item[0]:
            sparknum += item[1]  # spark计数
    return Sparknum, sparknum


def main():
    # 1 读取文件并处理
    file = 'README.md'
    result = load_and_deal(file)
    # 2 计数
    Sparknum,sparknum = count_words(result)
    # 3 输出
    print("the nums of Spark:" + str(Sparknum))  # 21
    print("the nums of spark:" + str(sparknum))  # 13

if __name__ == "__main__":
    conf = SparkConf().setMaster("local").setAppName("words count")
    sc = SparkContext(conf=conf)
    main()