from pyspark import SparkContext
from pyspark.sql import SQLContext
from pyspark.sql.types import *
def parse(line):
items = line.split(",")
return (long(items[0]), items[1], float(items[2]), int(items[3]), float(items[4]), float(items[5]))
if __name__ == "__main__":
sc = SparkContext(appName="CSV2Parquet")
sqlContext = SQLContext(sc)
schema = StructType([
StructField("col1", LongType(), True),
StructField("col2", StringType(), True),
StructField("col3", DoubleType(), True),
StructField("col4", IntegerType(), True),
StructField("col5", DoubleType(), True),
StructField("col6", DoubleType(), True)])
rdd = sc.textFile("./input.csv").map(parse)
df = sqlContext.createDataFrame(rdd, schema)
df.write.parquet('./input-parquet')
from pyspark.sql import SQLContext
from pyspark.sql.types import *
def parse(line):
items = line.split(",")
return (long(items[0]), items[1], float(items[2]), int(items[3]), float(items[4]), float(items[5]))
if __name__ == "__main__":
sc = SparkContext(appName="CSV2Parquet")
sqlContext = SQLContext(sc)
schema = StructType([
StructField("col1", LongType(), True),
StructField("col2", StringType(), True),
StructField("col3", DoubleType(), True),
StructField("col4", IntegerType(), True),
StructField("col5", DoubleType(), True),
StructField("col6", DoubleType(), True)])
rdd = sc.textFile("./input.csv").map(parse)
df = sqlContext.createDataFrame(rdd, schema)
df.write.parquet('./input-parquet')