使用python编写csv转parquet脚本

from pyspark import SparkContext
from pyspark.sql import SQLContext
from pyspark.sql.types import *


def parse(line):
    items = line.split(",")
    return (long(items[0]), items[1], float(items[2]), int(items[3]), float(items[4]), float(items[5]))


if __name__ == "__main__":
    sc = SparkContext(appName="CSV2Parquet")
    sqlContext = SQLContext(sc)


    schema = StructType([
            StructField("col1", LongType(), True),
            StructField("col2", StringType(), True),
            StructField("col3", DoubleType(), True),
            StructField("col4", IntegerType(), True),
            StructField("col5", DoubleType(), True),
            StructField("col6", DoubleType(), True)])




    rdd = sc.textFile("./input.csv").map(parse)
    df = sqlContext.createDataFrame(rdd, schema)
    df.write.parquet('./input-parquet')