spark之scala+java+maven+idea实现wordcount
1、Idea打开工作目录
2、新建模块,建java模块即可,选择相应的SDK
3、添加framework support,选择maven
在pom.xml中添加
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.3.0</version>
</dependency>
</dependencies>
刷新,会自动下载依赖包
4、若写scala代码,需添加scala支持:add framework support中选择scala即可。
5、创建scala class文件,kind选择object
Wordcount scala版本:
import org.apache.spark.{SparkConf, SparkContext}
/**
*
*/
object WordCountDemo {
// unit类似于java中的void
def main(args: Array[String]): Unit = {
//创建Spark配置对象
val conf = new SparkConf();
conf.setAppName("WordCountScala")
//设置master属性
conf.setMaster("local")
//conf.setMaster("spark://s201:7077")
//通过conf创建sc
val sc = new SparkContext(conf)
//加载文本文件
val rdd1 = sc.textFile("e:/sparkNote/word.txt")
//val rdd1 = sc.textFile(args(0));
//压扁
val rdd2 = rdd1.flatMap(line => line.split(" "))
//映射w => (w,1)
val rdd3 = rdd2.map((_,1))
val rdd4 = rdd3.reduceByKey(_ + _)
val r = rdd4.collect()
r.foreach(println)
}
}
6、wordcount java版本
package com.sxt ;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
/**
* Java版Spark单词统计程序
*/
public class WordCountJava {
public static void main(String[] args) {
SparkConf conf = new SparkConf();
conf.setAppName("WordCountScala");
//设置master属性
conf.setMaster("local");
//conf.setMaster("spark://s201:7077");
JavaSparkContext sc = new JavaSparkContext(conf);
//加载文件
JavaRDD<String> rdd1 = sc.textFile("e:/sparkNote/word.txt");
//JavaRDD<String> rdd1 = sc.textFile(args[0]);
//压扁
JavaRDD<String> rdd2 = rdd1.flatMap(new FlatMapFunction<String, String>() {
public Iterator<String> call(String s) throws Exception {
List<String> list = new ArrayList<String>();
String[] arr = s.split(" ");
for(String ss : arr){
list.add(ss) ;
}
return list.iterator() ;
}
});
//映射
JavaPairRDD<String,Integer> rdd3 = rdd2.mapToPair(new PairFunction<String, String, Integer>() {
public Tuple2<String, Integer> call(String s) throws Exception {
return new Tuple2<String, Integer>(s,1);
}
});
//聚合
JavaPairRDD<String,Integer> rdd4 = rdd3.reduceByKey(new Function2<Integer, Integer, Integer>() {
public Integer call(Integer v1, Integer v2) throws Exception {
return v1 + v2;
}
});
//收集,打印输出
for(Object o : rdd4.collect()){
System.out.println(o);
}
}
}