第一步:import org.apache.spark.{SparkConf, SparkContext}
第二步:val SparkConf = new SparkConf().setAppName("My_App").setMaster("local[2]")
第三步:val sc = new SparkContext(SparkConf)
第四步:sc.parallelize(List(1,2,3,4,5)).count
package com.ht.hadoop.service
import com.alibaba.fastjson.JSONArray
import com.ht.hadoop.util.ObjGenerator
import org.apache.spark.SparkContext
import org.apache.spark.sql.SparkSession
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
@Service
class WordCountService {
@Autowired
private val sparkSession: SparkSession = null
def wordCount(filePath: String): JSONArray = {
val sc: SparkContext = sparkSession.sparkContext
val fileRdd = sc.textFile(filePath).flatMap(_.split(" "))
.map((_, 1))
.reduceByKey(_ + _)
val wordCountArray: JSONArray = new JSONArray()
fileRdd.collect().foreach(x => {
wordCountArray.add(ObjGenerator.newJSON(Seq((x._1, x._2)): _*))
})
wordCountArray
}
}