ratings = spark.read.load("/FileStore/tables/u.data",format="csv", sep="\t", inferSchema="true", header="false")ratings = ratings.toDF(*['user_id', 'movie_id', 'rating', 'unix_timestamp'])
外观如下:
ratings.show()
import pyspark.sql.functions as F
我们的第一个函数F.col函数使我们可以访问列。
因此,如果我们想将一栏乘以2,可以将F.col用作:
ratings_with_scale10 = ratings.withColumn("ScaledRating", 2*F.col("rating"))ratings_with_scale10.show()
ratings_with_exp = ratings.withColumn("expRating", 2*F.exp("rating"))ratings_with_exp.show()
import pyspark.sql.functions as Ffrom pyspark.sql.types import *defsomefunc(value): if value < 3: return 'low' else: return 'high'#convert to a UDF Function by passing in the function and return type of function udfsomefunc = F.udf(somefunc, StringType())ratings_with_high_low = ratings.withColumn("high_low", udfsomefunc("rating"))ratings_with_high_low.show()
import mathfrom pyspark.sql import Rowdefrowwise_function(row): # convert row to dict: row_dict = row.asDict() # Add a new key in the dictionary with the new column name and value. row_dict['Newcol'] = math.exp(row_dict['rating']) # convert dict to row: newrow = Row(**row_dict) # return new row return newrow# convert ratings dataframe to RDDratings_rdd = ratings.rdd# apply our function to RDDratings_rdd_new = ratings_rdd.map(lambda row: rowwise_function(row))# Convert RDD Back to DataFrameratings_new_df = sqlContext.createDataFrame(ratings_rdd_new)ratings_new_df.show()
# Declare the schema for the output of our functionoutSchema = StructType([StructField('user_id',IntegerType(),True),StructField('movie_id',IntegerType(),True),StructField('rating',IntegerType(),True),StructField('unix_timestamp',IntegerType(),True),StructField('normalized_rating',DoubleType(),True)])# decorate our function with pandas_udf decorator@F.pandas_udf(outSchema, F.PandasUDFType.GROUPED_MAP)defsubtract_mean(pdf): # pdf is a pandas.DataFrame v = pdf.rating v = v - v.mean() pdf['normalized_rating'] =v return pdfrating_groupwise_normalization = ratings.groupby("movie_id").apply(subtract_mean)rating_groupwise_normalization.show()
# 0. Declare the schema for the output of our functionoutSchema = StructType([StructField('replication_id',IntegerType(),True),StructField('RMSE',DoubleType(),True)])# decorate our function with pandas_udf decorator@F.pandas_udf(outSchema, F.PandasUDFType.GROUPED_MAP)defrun_model(pdf): # 1. Get hyperparam values num_trees = pdf.num_trees.values[0] depth = pdf.depth.values[0] replication_id = pdf.replication_id.values[0] # 2. Train test split Xtrain,Xcv,ytrain,ycv = train_test_split..... # 3. Create model using the pandas dataframe clf = RandomForestRegressor(max_depth = depth, num_trees=num_trees,....) clf.fit(Xtrain,ytrain) # 4. Evaluate the model rmse = RMSE(clf.predict(Xcv,ycv) # 5. return results as pandas DF res =pd.DataFrame({'replication_id':replication_id,'RMSE':rmse}) return resresults = replicated_data.groupby("replication_id").apply(run_model)
以上只是一个想法,而不是一个有效的代码。
尽管应该稍作修改。
使用SQL
对于喜欢SQL的人,甚至可以使用SQL创建列。为此,我们需要注册一个临时SQL表,然后使用带有附加列的简单选择查询。一个人也可以用它来进行联接。
ratings.registerTempTable('ratings_table')newDF = sqlContext.sql('select *, 2*rating as newCol from ratings_table')newDF.show()
- END -
文源网络,仅供学习之用,如有侵权,联系删除。往期精彩◆ 50款开源工具你都用过吗?
◆ python+C、C++混合编程的应用
◆ python网络爬虫的基本原理详解
◆ Python自动操控excel,一小时解决你一天的工作
◆ 如何用Python增强Excel,减少处理复杂数据的痛苦?