spark执行hivesqlstr
法1:
spark = SparkSession \
.builder \
.appName("app_level") \
.enableHiveSupport() \
.getOrCreate()
spark.sql(sql)
法2:
os.system("export PYSPARK_PYTHON=python2.7.5")
appName = "VL_caltion"
mode = "yarn-client"
conf =SparkConf().setAppName(appName).setMaster("yarn-client")
sc = SparkContext(conf=conf)
hiveContext= HiveContext(sc)
self.result=sku_result.join(dept_result,["dc_id","seller_code","dept_code"]).coalesce(10)\
.select("seller_code","dept_code","sku_code","dc_id",F.when(sku_result.amt<10,dept_result.vl_mean_dept).otherwise(sku_result.vl_mean).alias("vl_mean"),
F.when(sku_result.amt<10,dept_result.vl_std_dept).otherwise(sku_result.vl_std).alias("vl_std"),
col("amt").alias("sku_order_count"),col("amt_dept").alias("dept_order_count"),
F.when(sku_result.amt<10,2).otherwise(1).alias("dim_type")).withColumn("dt",lit(endDate)).coalesce(10)
hiveContext.sql("set hive.exec.dynamic.partition.mode=nonstrict;set hive.exec.dynamic.partition=true;")
hiveContext.sql("alter table app.app_ivlt drop if existspartition(dt='"+endDate+"')")
self.result.write.mode("append").insertInto("app.app_ivlt ")