Spark培训之计算员工薪资

方法一、Scala：

 
 

  
  
val salaryRDD = sc.textFile("hdfs://192.168.1.171:9999/user/root/input/salaries/Salaries.csv").filter(!_.contains("Id"))
  
  
   
   

    
    def max(a: Double, b: Double): Double = {
   
   
   
   

    
    if (a > b) {
   
   
   
   

    
    return a
   
   
   
   

    
      } else {
   
   
   
   

    
    return b
   
   
   
   

    
      }
   
   
   
   

    
    }
   
   
  
  
salariesRDD.map(_.split(",")).filter(!_(0).contains("Id")).filter(!_(7).isEmpty).map(line => (line(2).trim.toUpperCase, line(7).toDouble)).reduceByKey(max).saveAsTextFile("hdfs://192.168.1.171:9999/user/root/input/salaries/max.txt")
 
 
 
 

  
  
  
  
方法
二
、DataFrame：
 
 
 
 

  
  
  
  
case class Salary(Id: Int, EmployeeName: String, JobTitle: String, BasePay: Double, OvertimePay: Double, OtherPay: Double, Benefits: Double, TotalPay: Double, TotalPayBenefits: Double, Year: String, Notes: String, Agency: String, Status: String)
  
  

 
 
 
 

  
  

   
   val salaryDF = salaryRDD.map(_.split(",")).map(p => Salary(p(0).trim.toInt, p(1), p(2), p(3).trim.toDouble, p(4).trim.toDouble, p(5).trim.toDouble, p(6).trim.toDouble, p(7).trim.toDouble, p(8).trim.toDouble, p(9), p(10), p(11), p(12))).toDF()
  
  
 
 
 
 

  
  
  
  
salaryDF.registerTempTable("salary")
  
  

   
   

    
    val salaries = sqlContext.sql("SELECT Year, SUM(TotalPay) FROM salary GROUP BY Year")
   
   
   
   

    
    salaries.map(t =>"Year: "+ t(0)).collect().foreach(println)
   
   
  
  
方法三、引用外部包：
 
 

 
 

  
  spark-shell --packages com.databricks:spark-csv_2.10:1.3.0
 
 

import org.apache.spark.sql.SQLContext

val sqlContext = new SQLContext(sc)
val df = sqlContext.load("com.databricks.spark.csv", Map("path" -> "hdfs://192.168.1.171:9999/user/root/input/salaries/Salaries.csv", "header" -> "true"))
df.select("EmployeeName").save("hdfs://192.168.1.171:9999/user/root/input/salaries/EmployeeName", "com.databricks.spark.csv")

方法四、SPARK-SQL：

spark-sql --packages com.databricks:spark-csv_2.10:1.3.0

CREATE TABLE salaries
USING com.databricks.spark.csv
OPTIONS (path "hdfs://192.168.1.171:9999/user/root/input/salaries/Salaries.csv", header "true");

  
  

   
   SELECT * FROM salaries A WHERE EmployeeName = 'Nelly S Fong';
  
  
  
  

   
   

    
    SELECT JobTitle, MAX(TotalPay) FROM salaries GROUP BY JobTitle;