pyspark中怎么在原有列基础上新生成一列
初次接触pyspark使用,对于表查询中的数据处理,在原有列数据基础上生成新的一列,可通过functions中的函数对数据进行处理
import sys
from encoder import decode_userfeature
reload(sys)
sys.setdefaultencoding('utf-8')
import yaml
import json
from urlparse import urlparse, parse_qs
from datetime import datetime, timedelta
from pytoolkit import TDWSQLProvider, TDWUtil
from pyspark.sql import SparkSession
from pyspark.sql import functions as f
from pyspark.sql.types import *
import base64
##################### global variables ##########################################
spark = SparkSession.builder.appName('CollectUserActions').getOrCreate()
##################### global variables ##########################################
def save_userdata(df, spark, datestr, dbname, tblname):
print('saving: %s : %s ...'%(tblname, datestr))
datepart = 'p_' + datestr
# create partition
tdw_util_out = TDWUtil(dbName=dbname, group='tl')
tdw_util_out.dropPartition(tblname, datepart) # 删除分区
tdw_util_out.createListPartition(tblname, datepart, datestr) # 创建分区
print('finish create new partition in table.')
# save to table
tdw = TDWSQLProvider(spark, db=dbname, group='tl') # 保存表
tdw.saveToTable(df, tblname, datepart)
rows = df.count()
print('finish write userdata: %s, rows: %s'%(datestr, rows))
# decode数据
def input_data(data):
res = decode_userfeature(data) # dict
return res
udf_user_info = f.udf(input_data, StringType())
# 处理原始数据和各标签
# 原始数据为字典形式的字符串,key为传入的参数,闭包思想传入传输
def udf_user_tag_rel(key):
def get_key(data):
tag_value = eval(str(data)).get(key)
return tag_value
return f.udf(get_key, StringType())
def process(conf, datestr):
datepart = 'p_' + datestr
tdw = TDWSQLProvider(spark, db=conf['log_db'], group='tl')
print('start processing...')
# 从feature中提取出age数据生成新的一列
df = tdw.table(conf['log_tb'], [datepart]). \
select('uin', udf_user_info('feature').alias('feature_dict')).withColumn('age', udf_user_tag_rel('age')('feature_dict')).\
select('uin', 'age')
df.show(50)
if __name__ == '__main__':
conf = yaml.load(open('****_conf.yaml')) # 读取yaml文件,返回一个对象,结果为一个字典
datestr = '*******' # 分区编号
process(conf, datestr)
print(datestr + ' end')