pyspark 将csv转为parquet格式

需要安装python3 和pyspark

手动配置数据转换格式

sc.stop()
from pyspark import SparkContext
from pyspark.sql import SQLContext
from pyspark.sql.types import *
def parse(line):
    items = line.split(",")
    return (items[0], items[1], items[2])
if __name__ == "__main__":
    sc = SparkContext(appName="CSV2Parquet")
    sqlContext = SQLContext(sc)
    schema = StructType([
        StructField("item_id", StringType(), True),
        StructField("time_interval", StringType(), True),
        StructField("invoice_id",IntegerType(), True)])
    rdd = sc.textFile("input.csv").map(parse)
    df = sqlContext.createDataFrame(rdd, schema)
    df.write.parquet('output.parquet')
    print("运行成功")

自动识别数据数据格式并转换

sc.stop()
from pyspark.sql import SparkSession
spark = SparkSession.builder.enableHiveSupport().getOrCreate()
df = spark.read.options(header='True', inferSchema='True', delimiter=',').csv("/Users/tanghc/Desktop/csv2parquet.csv")

#df.printSchema()
df.write.parquet('/Users/tanghc/Desktop/input-parquet135')
print("运行成功")
sc.stop()

读取parquet文件

sc.stop()
from pyspark import SparkContext, SparkConf
from pyspark.sql import SQLContext
from pyspark.sql.types import *
from pyspark.sql.session import SparkSession
if __name__ == "__main__":
    conf = SparkConf().setAppName('test_parquet')
    sc = SparkContext('local', 'test', conf=conf)
    spark = SparkSession(sc)
    df = spark.read.parquet("output.parquet")
    print(df.printSchema())
    print(df.first())
    print(df.show(5))
    print(df.count())
    print(type(df))
sc.stop()