RDD用法与实例（九）Join，Broadcast用法与对比

Join

products = sc.parallelize([(1, "Apple"), (2, "Orange"), (3, "TV"), (5, "Computer")])
trans = sc.parallelize([(1, (134, "OK")), (3, (34, "OK")), (5, (162, "Error")), (1, (135, "OK")), (2, (53, "OK")), (1, (45, "OK"))])

print(trans.join(products).take(20))

[(1, ((134, ‘OK’), ‘Apple’)), (1, ((135, ‘OK’), ‘Apple’)), (1, ((45, ‘OK’), ‘Apple’)), (2, ((53, ‘OK’), ‘Orange’)), (3, ((34, ‘OK’), ‘TV’)), (5, ((162, ‘Error’), ‘Computer’))]

broadcast

products = {1: "Apple", 2: "Orange", 3: "TV", 5: "Computer"}
trans = sc.parallelize([(1, (134, "OK")), (3, (34, "OK")), (5, (162, "Error")), (1, (135, "OK")), (2, (53, "OK")), (1, (45, "OK"))])

broadcasted_products = sc.broadcast(products)

results = trans.map(lambda x: (x[0], broadcasted_products.value[x[0]], x[1]))
#  results = trans.map(lambda x: (x[0], products[x[0]], x[1]))
print(results.take(20))

[(1, ‘Apple’, (134, ‘OK’)), (3, ‘TV’, (34, ‘OK’)), (5, ‘Computer’, (162, ‘Error’)), (1, ‘Apple’, (135, ‘OK’)), (2, ‘Orange’, (53, ‘OK’)), (1, ‘Apple’, (45, ‘OK’))]