#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import re
import sys
from operator import add
from pyspark import SparkConf, SparkContext
def compute_contribs(urls, rank):
"""
给urls计算
Args:
urls: 目标url相邻的urls集合
rank: 目标url的当前rank
Returns:
url: 相邻urls中的一个url
rank: 当前url的新的rank
"""
num_urls = len(urls)
for url in urls:
yield (url, rank / num_urls)
def split_url(url_line):
"""
把一行url切分开来
Args:
url_line: 一行url,如 1 2
Returns:
url, neighbor_url
"""
parts = re.split(r'\s+', url_line) # 正则
return parts[0], parts[1]
def compute_pagerank(sc, url_data_file, iterations):
"""
计算各个page的排名
Args:
sc: SparkContext
url_data_file: 测试数据文件
iterations: 迭代次数
Returns:
status: 成功就返回0
"""
# 读取url文件 ['1 2', '1 3', '2 1', '3 1']
lines = sc.textFile(url_data_file).map(lambda line: line.encode('utf8'))
# 建立Pair RDD (url, neighbor_urls) [(1,[2,3]), (2,[1]), (3, [1])]
links = lines.map(lambda line : split_url(line)).distinct().groupByKey().mapValues(lambda x: list(x)).cache()
# 初始化所有url的rank为1 [(1, 1), (2, 1), (3, 1)]
ranks = lines.map(lambda line : (line[0], 1))
for i in range(iterations):
# (url, [(neighbor_urls), rank]) join neighbor_urls and rank
# 把当前url的rank分别contribute到其他相邻的url (url, rank)
contribs = links.join(ranks).flatMap(
lambda url_urls_rank: compute_contribs(url_urls_rank[1][0], url_urls_rank[1][1])
)
# 把url的所有rank加起来,再赋值新的
ranks = contribs.reduceByKey(add).mapValues(lambda rank : rank * 0.85 + 0.15)
for (link, rank) in ranks.collect():
print("%s has rank %s." % (link, rank))
return 0
if __name__ == '__main__':
data_file = "/data/zz/data.txt"
iterations = 10
conf = SparkConf().setAppName('PythonPageRank')
sc = SparkContext(conf=conf)
ret = compute_pagerank(sc, data_file, iterations)
sys.exit(ret)
# coding:utf-8
from pyspark import SparkConf, SparkContext
def f(x):
list1 = []
s = len(x[1][0])
for y in x[1][0]:
list1.append(tuple((y, x[1][1]/s)))
return list1
if __name__== "__main__":
list = [('A', ('D',)), ('B', ('A',)), ('C', ('A', 'B')), ('D', ('A', 'C'))]
conf = SparkConf().setAppName('PythonPageRank').setMaster('local')
sc = SparkContext(conf=conf)
pages = sc.parallelize(list).map(lambda x: (x[0], tuple(x[1]))).partitionBy(4).cache()
# 初始pr值都设置为1
links = sc.parallelize(['A', 'B', 'C', 'D']).map(lambda x: (x, 1.0))
# join会把links和page按k合并,如('A',('D',))和('A',1.0) join之后变成 ('A', ('D',1.0))
# flatMap调用了f函数,并把结果平铺
rank = pages.join(links).flatMap(f)
# reduce
links = rank.reduceByKey(lambda x, y: x+y)
# 修正
links = links.mapValues(lambda x: 0.15+0.85*x)
links.saveAsTextFile("./pagerank")
j = pages.collect()
for i in j:
print(i)