import pandas as pd
from pyspark.sql import SparkSession
from pyspark.conf import SparkConf
from pyspark.sql.types import *
from pyspark.sql import functions as F, Window
初始化与配置环境
# 配置集群
config = SparkConf()
# config.set('spark.dynamicAllocation.maxExecutors', '8')
# config.set('spark.driver.memory', '4G')
# config.set('spark.executor.memory', '8G')
# config.set('spark.executor.cores', '8')
# config.set('spark.yarn.executora.memoryOverhead', '4G')
# config.set('spark.sql.shuffle.partitions', '500')
# config.set('spark.default.parallelism', '500')
# config.set('spark.port.maxRetries', '1000')
# config.set('spark.sql.sources.partitionOverwriteMode', 'dynamic')
config.set('spark.master','local[4]')
spark = SparkSession.builder.config(conf=config).getOrCreate()
创建DataFrame
字典方式创建
df = spark.createDataFrame([
{
'user_id': 'A203', 'country': 'India', 'browser': 'Chrome', 'OS': 'WIN', 'age': 33},
{
'user_id': 'A201', 'country': 'China', 'browser': 'Safari', 'OS': 'MacOs', 'age': 35},
{
'user_id': 'A205', 'country': 'UK', 'browser': 'Mozilla', 'OS': 'Linux', 'age': 25}
])
/usr/lib/spark/python/pyspark/sql/session.py:346: UserWarning: inferring schema from dict is deprecated,please use pyspark.sql.Row instead
warnings.warn("inferring schema from dict is deprecated,"
df.show()
+-----+---+-------+-------+-------+
| OS|age|browser|country|user_id|
+-----+---+-------+-------+-------+
| WIN| 33| Chrome| India| A203|
|MacOs| 35| Safari| China| A201|
|Linux| 25|Mozilla| UK| A205|
+-----+---+-------+-------+-------+
df.printSchema()
root
|-- OS: string (nullable = true)
|-- age: long (nullable = true)
|-- browser: string (nullable = true)
|-- country: string (nullable = true)
|-- user_id: string (nullable = true)
申明列类型创建
http://spark.apache.org/docs/latest/api/python/pyspark.sql.html#module-pyspark.sql.types