rdd转dataframe
from pyspark.sql import SparkSession
from pyspark.sql import Row
spark = SparkSession.builder.getOrCreate()
sc = spark.sparkContext
rdd= sc.parallelize([('John',30),('Mary',78)])
dataframe=spark.createDataFrame(rdd,['name','age'])
dataframe.show()
+----+---+
|name|age|
+----+---+
|John| 30|
|Mary| 78|
+----+---+
dataframe转rdd
rdd_ = dataframe.rdd.map(lambda t: (t.link_id, t.age))
rdd_.collect()
[(u'John', 30), (u'Mary', 78)]