第1关:数据清洗--过滤字段长度不足的且将出生日期转:
package com.yy
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
object edu{
/**********Begin**********/
// 此处可填写相关代码
case class Person(id:String,Name:String,CtfTp:String,CtfId:String,Gender:String,Birthday:String,Address:String,Zip:String,Duty:String,Mobile:String,Tel:String,Fax:String,EMail:String,Nation:String,Taste:String,Education:String,Company:String,Family:String,Version:String,Hotel:String,Grade:String,Duration:String,City:String)
/**********End**********/
def main(args: Array[String]): Unit = {
val spark = SparkSession
.builder()
.appName("Spark SQL")
.master("local")
.config("spark.some.config.option", "some-value")
.getOrCreate()
val rdd = spark.sparkContext.textFile("file:///root/files/part-00000-4ead9570-10e5-44dc-80ad-860cb072a9ff-c000.csv")
/**********Begin**********/
// 清洗脏数据(字段长度不足 23 的数据视为脏数据)
val rdd1: RDD[String] = rdd.filter(x=>{
val e=x.split(",",-1)