python 空指针排查_python-使用snowflake spark join器获得空指针异常

sfOptions = {

"sfURL" : "{}.snowflakecomputing.com".format(ACCOUNT_NAME),

"sfUser" : "{}@fmr.com".format(USER_ID),

"sfAccount" : "{}".format(ACCOUNT_ID),

"sfRole" : "{}".format(DEFAULT_ROLE),

"sfAuthenticator" : "oauth",

"sfToken" : "{}".format(oauth_token),

"sfDatabase" : "{}".format(DATABASE),

"sfSchema" : "{}".format(SCHEMA),

"sfWarehouse" : "{}".format(WAREHOUSE)

}

SNOWFLAKE_SOURCE_NAME = "net.snowflake.spark.snowflake"

....

conf = (SparkConf()

.setMaster("spark://")

.setAppName("Spark-Snowflake-Connector")

)

spark = (SparkSession.builder.config(conf=conf)

.enableHiveSupport()

.getOrCreate())

spark._jvm.net.snowflake.spark.snowflake.SnowflakeConnectorUtils.enablePushdownSession(spark._jvm.org.apache.spark.sql.SparkSession.builder().getOrCreate())

sdf = spark.read.format(SNOWFLAKE_SOURCE_NAME)

.options(**sfOptions)

.option("query", "select * from TIME_AGE")

.load()

sdf.show()