pyspark | 数据处理基本操作

import pandas as pd
from pyspark.sql import SparkSession
from pyspark.conf import SparkConf
from pyspark.sql.types import *
from pyspark.sql import functions as F, Window

初始化与配置环境

# 配置集群
config = SparkConf()
# config.set('spark.dynamicAllocation.maxExecutors', '8')
# config.set('spark.driver.memory', '4G')
# config.set('spark.executor.memory', '8G')
# config.set('spark.executor.cores', '8')
# config.set('spark.yarn.executora.memoryOverhead', '4G')
# config.set('spark.sql.shuffle.partitions', '500')
# config.set('spark.default.parallelism', '500')
# config.set('spark.port.maxRetries', '1000')
# config.set('spark.sql.sources.partitionOverwriteMode', 'dynamic')
config.set('spark.master','local[4]')

spark = SparkSession.builder.config(conf=config).getOrCreate()

创建DataFrame

字典方式创建

df = spark.createDataFrame([
    {
   'user_id': 'A203', 'country': 'India', 'browser': 'Chrome', 'OS': 'WIN', 'age': 33},
    {
   'user_id': 'A201', 'country': 'China', 'browser': 'Safari', 'OS': 'MacOs', 'age': 35},
    {
   'user_id': 'A205', 'country': 'UK', 'browser': 'Mozilla', 'OS': 'Linux', 'age': 25} 
])

/usr/lib/spark/python/pyspark/sql/session.py:346: UserWarning: inferring schema from dict is deprecated,please use pyspark.sql.Row instead
  warnings.warn("inferring schema from dict is deprecated,"

df.show()

+-----+---+-------+-------+-------+
|   OS|age|browser|country|user_id|
+-----+---+-------+-------+-------+
|  WIN| 33| Chrome|  India|   A203|
|MacOs| 35| Safari|  China|   A201|
|Linux| 25|Mozilla|     UK|   A205|
+-----+---+-------+-------+-------+

df.printSchema()

root
 |-- OS: string (nullable = true)
 |-- age: long (nullable = true)
 |-- browser: string (nullable = true)
 |-- country: string (nullable = true)
 |-- user_id: string (nullable = true)

申明列类型创建

http://spark.apache.org/docs/latest/api/python/pyspark.sql.html#module-pyspark.sql.types