'''
Created on 2018年9月22日
@author: Administrator
'''
from pyspark.conf import SparkConf
from pyspark.context import SparkContext
from pyspark.streaming.tests import result
from test.test_importlib.namespace_pkgs.both_portions.foo import one
from com.bjsxt.spark.wc import getCurrentSiteTop3User, getSiteIfo
def shows(s):
print(s)
def wordcount(line):
word=line.flatMap(lambda lines:lines.split(" "),True)
pairword=word.map(lambda words:(words,1),True)
result=pairword.reduceByKey(lambda v1,v2:(v1+v2),True)
result.foreach(lambda t:shows(t))
if __name__ == '__main__':
conf=SparkConf().setAppName("wordcount").setMaster("local")
sc=SparkContext(conf=conf)
line=sc.textFile("./wprds")
# 统计单词数量
wordcount(line)
版本二:
@author: Administrator
'''
from pyspark.conf import SparkConf
from pyspark.context import SparkContext
def show(s):
print(s)
def Wordcount(line):
rdd1=line.flatMap(lambda s:s.split(" ")).map(lambda s:(_,1)).reduceByKey(lambda v1,v2:(v1+v2)).foreach(lambda ss:show(ss))
if __name__ == '__main__':
conf=SparkConf().setMaster("local").setamAppName("test");
sc=SparkContext(conf=conf)
line=sc.textFile("./words")
#统计单词个数
Wordcount(line)